misc: Improve tagging script

This way we can push the tag *after* all the builds succeed. If
something goes wrong, we can always delete our local tag and try again.

.travis.yml

@@ -24,21 +24,21 @@ install: 'make deps'
 matrix:
   fast_finish: false
   allow_failures:
-  - go: 1.11.x
+  - go: 1.12.x
   - go: tip
   - os: osx
   # include only one build for osx for a quicker build as the nr. of these runners are sparse
   include:
   - name: "basic tests"
-    go: 1.10.x
+    go: 1.11.x
     env: TEST_BLOCK=basic
   - name: "shell tests"
-    go: 1.10.x
+    go: 1.11.x
     env: TEST_BLOCK=shell
   - name: "race tests"
-    go: 1.10.x
+    go: 1.11.x
     env: TEST_BLOCK=race
-  - go: 1.11.x
+  - go: 1.12.x
   - go: tip
   - os: osx
 script: 'TEST_BLOCK="$TEST_BLOCK" make test'

Makefile

@@ -16,11 +16,16 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 SHELL = /usr/bin/env bash
-.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms copr tag release funcgen
+.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs
+.PHONY: rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag
+.PHONY: mkosi mkosi_fedora-30 mkosi_fedora-29 mkosi_debian-10 mkosi_ubuntu-bionic mkosi_archlinux
+.PHONY: release releases_path release_fedora-30 release_fedora-29 release_debian-10 release_ubuntu-bionic release_archlinux
+.PHONY: funcgen
 .SILENT: clean bindata
 
 # a large amount of output from this `find`, can cause `make` to be much slower!
 GO_FILES := $(shell find * -name '*.go' -not -path 'old/*' -not -path 'tmp/*')
+MCL_FILES := $(shell find lang/funcs/ -name '*.mcl' -not -path 'old/*' -not -path 'tmp/*')
 
 SVERSION := $(or $(SVERSION),$(shell git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --tags --dirty --always))
 VERSION := $(or $(VERSION),$(shell git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --tags --abbrev=0))
@@ -48,9 +53,23 @@ GOOSARCHES ?= linux/amd64 linux/ppc64 linux/ppc64le linux/arm64 darwin/amd64
 GOHOSTOS = $(shell go env GOHOSTOS)
 GOHOSTARCH = $(shell go env GOHOSTARCH)
 
-RPM_PKG = releases/$(VERSION)/rpm/mgmt-$(VERSION)-1.x86_64.rpm
-DEB_PKG = releases/$(VERSION)/deb/mgmt_$(VERSION)_amd64.deb
-PACMAN_PKG = releases/$(VERSION)/pacman/mgmt-$(VERSION)-1-x86_64.pkg.tar.xz
+TOKEN_FEDORA-30 = fedora-30
+TOKEN_FEDORA-29 = fedora-29
+TOKEN_DEBIAN-10 = debian-10
+TOKEN_UBUNTU-BIONIC = ubuntu-bionic
+TOKEN_ARCHLINUX = archlinux
+
+FILE_FEDORA-30 = mgmt-$(TOKEN_FEDORA-30)-$(VERSION)-1.x86_64.rpm
+FILE_FEDORA-29 = mgmt-$(TOKEN_FEDORA-29)-$(VERSION)-1.x86_64.rpm
+FILE_DEBIAN-10 = mgmt_$(TOKEN_DEBIAN-10)_$(VERSION)_amd64.deb
+FILE_UBUNTU-BIONIC = mgmt_$(TOKEN_UBUNTU-BIONIC)_$(VERSION)_amd64.deb
+FILE_ARCHLINUX = mgmt-$(TOKEN_ARCHLINUX)-$(VERSION)-1-x86_64.pkg.tar.xz
+
+PKG_FEDORA-30 = releases/$(VERSION)/$(TOKEN_FEDORA-30)/$(FILE_FEDORA-30)
+PKG_FEDORA-29 = releases/$(VERSION)/$(TOKEN_FEDORA-29)/$(FILE_FEDORA-29)
+PKG_DEBIAN-10 = releases/$(VERSION)/$(TOKEN_DEBIAN-10)/$(FILE_DEBIAN-10)
+PKG_UBUNTU-BIONIC = releases/$(VERSION)/$(TOKEN_UBUNTU-BIONIC)/$(FILE_UBUNTU-BIONIC)
+PKG_ARCHLINUX = releases/$(VERSION)/$(TOKEN_ARCHLINUX)/$(FILE_ARCHLINUX)
 
 SHA256SUMS = releases/$(VERSION)/SHA256SUMS
 SHA256SUMS_ASC = $(SHA256SUMS).asc
@@ -131,7 +150,7 @@ lang: ## generates the lexer/parser for the language frontend
 $(PROGRAM): build/mgmt-${GOHOSTOS}-${GOHOSTARCH} ## build an mgmt binary for current host os/arch
 	cp -a $< $@
 
-$(PROGRAM).static: $(GO_FILES)
+$(PROGRAM).static: $(GO_FILES) $(MCL_FILES)
 	@echo "Building: $(PROGRAM).static, version: $(SVERSION)..."
 	go generate
 	go build -a -installsuffix cgo -tags netgo -ldflags '-extldflags "-static" -X main.program=$(PROGRAM) -X main.version=$(SVERSION) -s -w' -o $(PROGRAM).static $(BUILD_FLAGS);
@@ -146,10 +165,10 @@ build-debug: $(PROGRAM)
 # extract os and arch from target pattern
 GOOS=$(firstword $(subst -, ,$*))
 GOARCH=$(lastword $(subst -, ,$*))
-build/mgmt-%: $(GO_FILES) | bindata lang funcgen
+build/mgmt-%: $(GO_FILES) $(MCL_FILES) | bindata lang funcgen
 	@echo "Building: $(PROGRAM), os/arch: $*, version: $(SVERSION)..."
 	@# reassigning GOOS and GOARCH to make build command copy/pastable
-	@# go 1.10 requires specifying the package for ldflags
+	@# go 1.10+ requires specifying the package for ldflags
 	@if go version | grep -qE 'go1.9'; then \
 		time env GOOS=${GOOS} GOARCH=${GOARCH} go build -i -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS); \
 	else \
@@ -164,6 +183,7 @@ clean: ## clean things up
 	$(MAKE) --quiet -C bindata clean
 	$(MAKE) --quiet -C lang/funcs clean
 	$(MAKE) --quiet -C lang clean
+	$(MAKE) --quiet -C misc/mkosi clean
 	rm -f lang/funcs/core/generated_funcs.go || true
 	rm -f lang/funcs/core/generated_funcs_test.go || true
 	[ ! -e $(PROGRAM) ] || rm $(PROGRAM)
@@ -326,6 +346,10 @@ upload-rpms: rpmbuild/RPMS/ rpmbuild/RPMS/SHA256SUMS rpmbuild/RPMS/SHA256SUMS.as
 		rsync -avz --prune-empty-dirs rpmbuild/RPMS/ $(SERVER):$(REMOTE_PATH)/RPMS/; \
 	fi
 
+upload-releases:
+	echo Running releases/ upload...
+	rsync -avz --exclude '.mkdir' --exclude 'mgmt-release.url' releases/ $(SERVER):$(REMOTE_PATH)/releases/
+
 #
 #	copr build
 #
@@ -338,18 +362,57 @@ copr: upload-srpms ## build in copr
 tag: ## tags a new release
 	./misc/tag.sh
 
+#
+#	mkosi
+#
+mkosi: mkosi_fedora-30 mkosi_fedora-29 mkosi_debian-10 mkosi_ubuntu-bionic mkosi_archlinux ## builds distro packages via mkosi
+
+mkosi_fedora-30: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_fedora-29: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_debian-10: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_ubuntu-bionic: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_archlinux: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
 #
 #	release
 #
 release: releases/$(VERSION)/mgmt-release.url ## generates and uploads a release
 
-releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA256SUMS_ASC)
+releases_path:
+	@#Don't put any other output or dependencies in here or they'll show!
+	@echo "releases/$(VERSION)/"
+
+release_fedora-30: $(PKG_FEDORA-30)
+release_fedora-29: $(PKG_FEDORA-29)
+release_debian-10: $(PKG_DEBIAN-10)
+release_ubuntu-bionic: $(PKG_UBUNTU-BIONIC)
+release_archlinux: $(PKG_ARCHLINUX)
+
+releases/$(VERSION)/mgmt-release.url: $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_DEBIAN-10) $(PKG_UBUNTU-BIONIC) $(PKG_ARCHLINUX) $(SHA256SUMS_ASC)
+	@echo "Pushing git tag $(VERSION) to origin..."
+	git push origin $(VERSION)
 	@echo "Creating github release..."
 	hub release create \
 		-F <( echo -e "$(VERSION)\n";echo "Verify the signatures of all packages before you use them. The signing key can be downloaded from https://purpleidea.com/contact/#pgp-key to verify the release." ) \
-		-a $(RPM_PKG) \
-		-a $(DEB_PKG) \
-		-a $(PACMAN_PKG) \
+		-a $(PKG_FEDORA-30) \
+		-a $(PKG_FEDORA-29) \
+		-a $(PKG_DEBIAN-10) \
+		-a $(PKG_UBUNTU-BIONIC) \
+		-a $(PKG_ARCHLINUX) \
 		-a $(SHA256SUMS_ASC) \
 		$(VERSION) \
 		> releases/$(VERSION)/mgmt-release.url \
@@ -357,32 +420,48 @@ releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA2
 		|| rm -f releases/$(VERSION)/mgmt-release.url
 
 releases/$(VERSION)/.mkdir:
-	mkdir -p releases/$(VERSION)/{deb,rpm,pacman}/ && touch releases/$(VERSION)/.mkdir
+	mkdir -p releases/$(VERSION)/{$(TOKEN_FEDORA-30),$(TOKEN_FEDORA-29),$(TOKEN_DEBIAN-10),$(TOKEN_UBUNTU-BIONIC),$(TOKEN_ARCHLINUX)}/ && touch releases/$(VERSION)/.mkdir
 
-releases/$(VERSION)/rpm/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
-	@echo "Generating: rpm changelog..."
-	./misc/make-rpm-changelog.sh $(VERSION)
+releases/$(VERSION)/$(TOKEN_FEDORA-30)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-rpm-changelog.sh "$${distro}" $(VERSION)
 
-$(RPM_PKG): releases/$(VERSION)/rpm/changelog
-	@echo "Building: rpm package..."
-	./misc/fpm-pack.sh rpm $(VERSION) libvirt-devel augeas-devel
+$(PKG_FEDORA-30): releases/$(VERSION)/$(TOKEN_FEDORA-30)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_FEDORA-30)" libvirt-devel augeas-devel
 
-releases/$(VERSION)/deb/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
-	@echo "Generating: deb changelog..."
-	./misc/make-deb-changelog.sh $(VERSION)
+releases/$(VERSION)/$(TOKEN_FEDORA-29)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-rpm-changelog.sh "$${distro}" $(VERSION)
 
-$(DEB_PKG): releases/$(VERSION)/deb/changelog
-	@echo "Building: deb package..."
-	./misc/fpm-pack.sh deb $(VERSION) libvirt-dev libaugeas-dev
+$(PKG_FEDORA-29): releases/$(VERSION)/$(TOKEN_FEDORA-29)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_FEDORA-29)" libvirt-devel augeas-devel
 
-$(PACMAN_PKG): $(PROGRAM) releases/$(VERSION)/.mkdir
-	@echo "Building: pacman package..."
-	./misc/fpm-pack.sh pacman $(VERSION) libvirt augeas
+releases/$(VERSION)/$(TOKEN_DEBIAN-10)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-deb-changelog.sh "$${distro}" $(VERSION)
 
-$(SHA256SUMS): $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG)
+$(PKG_DEBIAN-10): releases/$(VERSION)/$(TOKEN_DEBIAN-10)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_DEBIAN-10)" libvirt-dev libaugeas-dev
+
+releases/$(VERSION)/$(TOKEN_UBUNTU-BIONIC)/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-deb-changelog.sh "$${distro}" $(VERSION)
+
+$(PKG_UBUNTU-BIONIC): releases/$(VERSION)/$(TOKEN_UBUNTU-BIONIC)/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_UBUNTU-BIONIC)" libvirt-dev libaugeas-dev
+
+$(PKG_ARCHLINUX): $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) "$(FILE_ARCHLINUX)" libvirt augeas
+
+$(SHA256SUMS): $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_DEBIAN-10) $(PKG_UBUNTU-BIONIC) $(PKG_ARCHLINUX)
 	@# remove the directory separator in the SHA256SUMS file
 	@echo "Generating: sha256 sum..."
-	sha256sum $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) | awk -F '/| ' '{print $$1"  "$$6}' > $(SHA256SUMS)
+	sha256sum $(PKG_FEDORA-30) $(PKG_FEDORA-29) $(PKG_DEBIAN-10) $(PKG_UBUNTU-BIONIC) $(PKG_ARCHLINUX) | awk -F '/| ' '{print $$1"  "$$6}' > $(SHA256SUMS)
 
 $(SHA256SUMS_ASC): $(SHA256SUMS)
 	@echo "Signing sha256 sum..."

README.md

@@ -9,6 +9,56 @@
 [![Patreon](https://img.shields.io/badge/patreon-donate-yellow.svg?style=flat-square)](https://www.patreon.com/purpleidea)
 [![Liberapay](https://img.shields.io/badge/liberapay-donate-yellow.svg?style=flat-square)](https://liberapay.com/purpleidea/donate)
 
+## About:
+
+`Mgmt` is a real-time automation tool. It is familiar to existing configuration
+management software, but is drastically more powerful as it can allow you to
+build real-time, closed-loop feedback systems, in a very safe way, and with a
+surprisingly small amout of our `mcl` code. For example, the following code will
+ensure that your file server is set to read-only when it's friday.
+
+```mcl
+import "datetime"
+$is_friday = datetime.weekday(datetime.now()) == "friday"
+file "/srv/files/" {
+	state => "exists",
+	mode => if $is_friday { # this updates the mode, the instant it changes!
+		"0550"
+	} else {
+		"0770"
+	},
+}
+```
+
+It can run continuously, intermittently, or on-demand, and in the first case, it
+will guarantee that your system is always in the desired state for that instant!
+In this mode it can run as a decentralized cluster of agents across your
+network, each exchanging information with the others in real-time, to respond to
+your changing needs. For example, if you want to ensure that some resource runs
+on a maximum of two hosts in your cluster, you can specify that as well:
+
+```mcl
+import "sys"
+import "world"
+
+# we'll set a few scheduling options:
+$opts = struct{strategy => "rr", max => 2, ttl => 10,}
+
+# schedule in a particular namespace with options:
+$set = world.schedule("xsched", $opts)
+
+if sys.hostname() in $set {
+	# use your imagination to put something more complex right here...
+	print "i got scheduled" {} # this will run on the chosen machines
+}
+```
+
+As you add and remove hosts from the cluster, the real-time `schedule` function
+will dynamically pick up to two hosts from the available pool. These specific
+functions aren't intrinsic to the core design, and new ones can be easily added.
+
+Please read on if you'd like to learn more...
+
 ## Community:
 
 Come join us in the `mgmt` community!
@@ -30,7 +80,7 @@ approach. The project contains an engine and a language.
 
 Mgmt is a fairly new project. It is usable today, but not yet feature complete.
 With your help you'll be able to influence our design and get us to 1.0 sooner!
-Interested developers should read the [quick start guide](docs/quick-start-guide.md).
+Interested users should read the [quick start guide](docs/quick-start-guide.md).
 
 ## Documentation:
 
@@ -38,7 +88,7 @@ Please read, enjoy and help improve our documentation!
 
 | Documentation | Additional Notes |
 |---|---|
-| [quick start guide](docs/quick-start-guide.md) | for mgmt developers |
+| [quick start guide](docs/quick-start-guide.md) | for everyone |
 | [frequently asked questions](docs/faq.md) | for everyone |
 | [general documentation](docs/documentation.md) | for everyone |
 | [language guide](docs/language-guide.md) | for everyone |
@@ -57,22 +107,18 @@ If you have a well phrased question that might benefit others, consider asking
 it by sending a patch to the [FAQ](docs/faq.md) section. I'll merge your
 question, and a patch with the answer!
 
-## Roadmap:
+## Get involved:
 
 Feel free to grab one of the straightforward [#mgmtlove](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 issues if you're a first time contributor to the project or if you're unsure
-about what to hack on!
-Please see: [TODO.md](TODO.md) for a list of upcoming work and TODO items.
-Please get involved by working on one of these items or by suggesting something
-else!
+about what to hack on! Please get involved by working on one of these items or
+by suggesting something else! There are some lower priority issues and harder
+issues available in our [TODO](TODO.md) file. Please have a look.
 
 ## Bugs:
 
 Please set the `DEBUG` constant in [main.go](https://github.com/purpleidea/mgmt/blob/master/main.go)
 to `true`, and post the logs when you report the [issue](https://github.com/purpleidea/mgmt/issues).
-Bonus points if you provide a [shell](https://github.com/purpleidea/mgmt/tree/master/test/shell)
-or [OMV](https://github.com/purpleidea/mgmt/tree/master/test/omv) reproducible
-test case.
 Feel free to read my article on [debugging golang programs](https://purpleidea.com/blog/2016/02/15/debugging-golang-programs/).
 
 ## Patches:

TODO.md

@@ -1,10 +1,18 @@
 # TODO
 
-If you're looking for something to do, look here!
-Let us know if you're working on one of the items.
-If you'd like something to work on, ping @purpleidea and I'll create an issue
-tailored especially for you! Just let me know your approximate golang skill
-level and how many hours you'd like to spend on the patch.
+Here is a TODO list of longstanding items that are either lower-priority, or
+more involved in terms of time, skill-level, and/or motivation.
+
+Please have a look, and let us know if you're working on one of the items. It's
+best to open an issue to track your progress and to discuss any implementation
+questions you might have.
+
+Lastly, if you'd like something different to work on, please ping @purpleidea
+and I'll create an issue tailored especially for your approximate golang skill
+level and available time commitment in terms of hours you'd need to spend on the
+patch.
+
+Happy Hacking!
 
 ## Package resource
 
@@ -19,7 +27,7 @@ level and how many hours you'd like to spend on the patch.
 
 ## Svc resource
 
-- [ ] base resource improvements
+- [ ] refreshonly support [:heart:](https://github.com/purpleidea/mgmt/issues/464)
 
 ## Exec resource
 
@@ -33,33 +41,14 @@ level and how many hours you'd like to spend on the patch.
 
 - [ ] automatic edges to file resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 
-## Virt (libvirt) resource
-
-- [ ] base resource improvements [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-
-## Net (systemd-networkd) resource
-
-- [ ] base resource
-
-## Nspawn (systemd-nspawn) resource
-
-- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-
-## Mount (systemd-mount) resource
-
-- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-
-## Cron (systemd-timer) resource
-
-- [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
-
 ## Http resource
 
 - [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove)
 
 ## Etcd improvements
 
-- [ ] fix embedded etcd master race
+- [ ] fix etcd race bug that only happens during CI testing (intermittently
+failing test case issue)
 
 ## Torrent/dht file transfer
 
@@ -69,17 +58,33 @@ level and how many hours you'd like to spend on the patch.
 
 - [ ] base plumbing
 
+## Resource improvements
+
+- [ ] more reversible resources implemented
+- [ ] more "cloud" resources
+
 ## Language improvements
 
 - [ ] more core functions
 - [ ] automatic language formatter, ala `gofmt`
 - [ ] gedit/gnome-builder/gtksourceview syntax highlighting
 - [ ] vim syntax highlighting
-- [x] emacs syntax highlighting: see `misc/emacs/`
+- [ ] emacs syntax highlighting: see `misc/emacs/` (needs updating)
+- [ ] exposed $error variable for feedback in the language
+- [ ] improve the printf function to add %[]s, %[]f ([]str, []float) and map,
+struct, nested etc... %v would be nice too!
+- [ ] add line/col/file annotations to AST so we can get locations of errors
+that the parser finds
+- [ ] add more error messages with the `%error` pattern in parser.y
+- [ ] we should have helper functions or language sugar to pull a field out of a
+struct, or a value out of a map, or an index out of a list, etc...
+
+## Engine improvements
+
+- [ ] add a "waiting for func" message in the func engine to notify the user
+about slow functions...
 
 ## Other
 
-- [ ] better error/retry handling
-- [ ] deb package target in Makefile
 - [ ] reproducible builds
 - [ ] add your suggestions!

docs/development.md

@@ -5,6 +5,118 @@ developing `mgmt`. Useful tools, conventions, etc.
 
 Be sure to read [quick start guide](quick-start-guide.md) first.
 
+## Vagrant
+
+If you would like to avoid doing the above steps manually, we have prepared a
+[Vagrant](https://www.vagrantup.com/) environment for your convenience. From the
+project directory, run a `vagrant up`, and then a `vagrant status`. From there,
+you can `vagrant ssh` into the `mgmt` machine. The `MOTD` will explain the rest.
+This environment isn't commonly used by the `mgmt` developers, so it might not
+be working properly.
+
+## Using Docker
+
+Alternatively, you can check out the [docker-guide](docker-guide.md) in order to
+develop or deploy using docker. This method is not endorsed or supported, so use
+at your own risk, as it might not be working properly.
+
+## Information about dependencies
+
+Software projects have a few different kinds of dependencies. There are _build_
+dependencies, _runtime_ dependencies, and additionally, a few extra dependencies
+required for running the _test_ suite.
+
+### Build
+
+* `golang` 1.11 or higher (required, available in some distros and distributed
+as a binary officially by [golang.org](https://golang.org/dl/))
+
+### Runtime
+
+A relatively modern GNU/Linux system should be able to run `mgmt` without any
+problems. Since `mgmt` runs as a single statically compiled binary, all of the
+library dependencies are included. It is expected, that certain advanced
+resources require host specific facilities to work. These requirements are
+listed below:
+
+| Resource | Dependency        | Version                     | Check version with                                        |
+|----------|-------------------|-----------------------------|-----------------------------------------------------------|
+| augeas   | augeas-devel      | `augeas 1.6` or greater     | `dnf info augeas-devel` or `apt-cache show libaugeas-dev` |
+| file     | inotify           | `Linux 2.6.27` or greater   | `uname -a`                                                |
+| hostname | systemd-hostnamed | `systemd 25` or greater     | `systemctl --version`                                     |
+| nspawn   | systemd-nspawn    | `systemd ???` or greater    | `systemctl --version`                                     |
+| pkg      | packagekitd       | `packagekit 1.x` or greater | `pkcon --version`                                         |
+| svc      | systemd           | `systemd ???` or greater    | `systemctl --version`                                     |
+| virt     | libvirt-devel     | `libvirt 1.2.0` or greater  | `dnf info libvirt-devel` or `apt-cache show libvirt-dev`  |
+| virt     | libvirtd          | `libvirt 1.2.0` or greater  | `libvirtd --version`                                      |
+
+For building a visual representation of the graph, `graphviz` is required.
+
+To build `mgmt` without augeas support please run:
+`GOTAGS='noaugeas' make build`
+
+To build `mgmt` without libvirt support please run:
+`GOTAGS='novirt' make build`
+
+To build `mgmt` without docker support please run:
+`GOTAGS='nodocker' make build`
+
+To build `mgmt` without augeas, libvirt or docker support please run:
+`GOTAGS='noaugeas novirt nodocker' make build`
+
+## OSX/macOS/Darwin development
+
+Developing and running `mgmt` on macOS is currently not supported (but not
+discouraged either). Meaning it might work but in the case it doesn't you would
+have to provide your own patches to fix problems (the project maintainer and
+community are glad to assist where needed).
+
+There are currently some issues that make `mgmt` less suitable to run for
+provisioning macOS. But as a client to provision remote servers it should run
+fine.
+
+Since the primary supported systems are Linux and these are the environments
+tested, it is wise to run these suites during macOS development as well. To ease
+this, Docker can be leveraged ([Docker for Mac](https://docs.docker.com/docker-for-mac/)).
+
+Before running any of the commands below create the development Docker image:
+
+```
+docker/scripts/build-development
+```
+
+This image requires updating every time dependencies (`make-deps.sh`) changes.
+
+Then to run the test suite:
+
+```
+docker run --rm -ti \
+	-v $PWD:/go/src/github.com/purpleidea/mgmt/ \
+	-w /go/src/github.com/purpleidea/mgmt/ \
+	purpleidea/mgmt:development \
+	make test
+```
+
+For convenience this command is wrapped in `docker/scripts/exec-development`.
+
+Basically any command can be executed this way. Because the repository source is
+mounted into the Docker container invocation will be quick and allow rapid
+testing, for example:
+
+```
+docker/scripts/exec-development test/test-shell.sh load0.sh
+```
+
+Other examples:
+
+```
+docker/scripts/exec-development make build
+docker/scripts/exec-development ./mgmt run --tmp-prefix lang examples/lang/load0.mcl
+```
+
+Be advised that this method is not supported and it might not be working
+properly.
+
 ## Testing
 
 This project has both unit tests in the form of golang tests and integration
@@ -45,5 +157,6 @@ individual tests to run.
 
 ### IDE/Editor support
 
-- Emacs: see `misc/emacs/`
-- [Textmate](https://github.com/aequitas/mgmt.tmbundle)
+* Emacs: see `misc/emacs/`
+* [Textmate](https://github.com/aequitas/mgmt.tmbundle)
+* [VSCode](https://github.com/aequitas/mgmt.vscode)

docs/documentation.md

@@ -147,7 +147,7 @@ Invoke `mgmt` with the `--puppet` switch, which supports 3 variants:
 
 	`mgmt run puppet --puppet 'file { "/etc/ntp.conf": ensure => file }'`
 
-For more details and caveats see [Puppet.md](Puppet.md).
+For more details and caveats see [puppet-guide.md](puppet-guide.md).
 
 #### Blog post
 
@@ -250,6 +250,43 @@ integer, then that value is the max size for that semaphore. Valid semaphore
 id's include: `some_id`, `hello:42`, `not:smart:4` and `:13`. It is expected
 that the last bare example be only used by the engine to add a global semaphore.
 
+#### Rewatch
+
+Boolean. Rewatch specifies whether we re-run the Watch worker during a graph
+swap if it has errored. When doing a graph compare to swap the graphs, if this
+is true, and this particular worker has errored, then we'll remove it and add it
+back as a new vertex, thus causing it to run again. This is different from the
+`Retry` metaparam which applies during the normal execution. It is only when
+this is exhausted that we're in permanent worker failure, and only then can we
+rely on this metaparam.
+
+#### Realize
+
+Boolean. Realize ensures that the resource is guaranteed to converge at least
+once before a potential graph swap removes or changes it. This guarantee is
+useful for fast changing graphs, to ensure that the brief creation of a resource
+is seen. This guarantee does not prevent against the engine quitting normally,
+and it can't guarantee it if the resource is blocked because of a failed
+pre-requisite resource.
+*XXX: This is currently not implemented!*
+
+#### Reverse
+
+Boolean. Reverse is a property that some resources can implement that specifies
+that some "reverse" operation should happen when that resource "disappears". A
+disappearance happens when a resource is defined in one instance of the graph,
+and is gone in the subsequent one. This disappearance can happen if it was
+previously in an if statement that then becomes false.
+
+This is helpful for building robust programs with the engine. The engine adds a
+"reversed" resource to that subsequent graph to accomplish the desired "reverse"
+mechanics. The specifics of what this entails is a property of the particular
+resource that is being "reversed".
+
+It might be wise to combine the use of this meta parameter with the use of the
+`realize` meta parameter to ensure that your reversed resource actually runs at
+least once, if there's a chance that it might be gone for a while.
+
 ### Lang metadata file
 
 Any module *must* have a metadata file in its root. It must be named
@@ -298,10 +335,6 @@ recommended that you use this, since it's preferable to write code in the
 The main interface to the `mgmt` tool is the command line. For the most recent
 documentation, please run `mgmt --help`.
 
-#### `--yaml <graph.yaml>`
-
-Point to a graph file to run.
-
 #### `--converged-timeout <seconds>`
 
 Exit if the machine has converged for approximately this many seconds.

docs/faq.md

@@ -212,9 +212,48 @@ requires a number of seconds as an argument.
 #### Example:
 
 ```
-./mgmt run lang --lang examples/lang/hello0.mcl --converged-timeout=5
+./mgmt run lang examples/lang/hello0.mcl --converged-timeout=5
 ```
 
+### When I try to build `mgmt` I see: `no Go files in $GOPATH/src/github.com/purpleidea/mgmt/bindata`.
+
+Due to the arcane way that `golang` designed its `$GOPATH`, the main project
+directory must be inside your `$GOPATH`, and at the appropriate FQDN. This is:
+`$GOPATH/src/github.com/purpleidea/mgmt/`. If you have your project root outside
+of that directory, then you may get this error when you try to build it. In this
+case there is likely a `go get` version of the project at this location. Remove
+it and replace it with your git cloned directory. In my case, I like to work on
+things in `~/code/mgmt/`, so that path is a symlink that points to the long
+project directory.
+
+### Why does my file resource error with `no such file or directory`?
+
+If you create a file resource and only specify the content like this:
+
+```
+file "/tmp/foo" {
+	content => "hello world\n",
+}
+```
+
+Then this will attempt to set the contents of that file to the desired string,
+but *only* if that file already exists. If you'd like to ensure that it also
+gets created in case it is not present, then you must also specify the state:
+
+```
+file "/tmp/foo" {
+	state => "exists",
+	content => "hello world\n",
+}
+```
+
+Similar logic applies for situations when you only specify the `mode` parameter.
+
+This all turns out to be more safe and "correct", in that it would error and
+prevent masking an error for a situation when you expected a file to already be
+at that location. It also turns out to simplify the internals significantly, and
+remove an ambiguous scenario with the reversable file resource.
+
 ### On startup `mgmt` hangs after: `etcd: server: starting...`.
 
 If you get an error message similar to:
@@ -255,7 +294,7 @@ serious automation workloads. Support for non-Linux operating systems isn't a
 high priority of mine, but we're happy to accept patches for missing features
 or resources that you think would make sense on your favourite platform.
 
-### Why aren't you using `glide` or `godep` for dependency management?
+### Why aren't you using `glide`, `godep` or `go mod` for dependency management?
 
 Vendoring dependencies means that as the git master branch of each dependency
 marches on, you're left behind using an old version. As a result, bug fixes and

docs/function-guide.md

@@ -219,7 +219,7 @@ Init(init *interfaces.Init) error
 
 This is called to initialize the function. If something goes wrong, it should
 return an error. It is passed a struct that contains all the important
-information and poiinters that it might need to work with throughout its
+information and pointers that it might need to work with throughout its
 lifetime. As a result, it will need to save a copy to that pointer for future
 use in the other methods.
 
@@ -377,9 +377,9 @@ might be different ways you would want to call `printf`, such as:
 `printf("the %s is %d", "answer", 42)` or `printf("3 * 2 = %d", 3 * 2)`. Since
 you couldn't implement the infinite number of possible signatures, this API lets
 you write code which can be coerced into different forms. This makes
-implementing what would appear to be generic or polymorphic, instead something
-that is actually static and that still has the static type safety properties
-that were guaranteed by the mgmt language.
+implementing what would appear to be generic or polymorphic, instead of
+something that is actually static and that still has the static type safety
+properties that were guaranteed by the mgmt language.
 
 Since this is an advanced topic, it is not described in full at this time. For
 more information please have a look at the source code comments, some of the

docs/language-guide.md

@@ -511,6 +511,9 @@ without making any changes. The `ExprVar` node naturally consumes scope's and
 the `StmtProg` node cleverly passes the scope through in the order expected for
 the out-of-order bind logic to work.
 
+This step typically calls the ordering algorithm to determine the correct order
+of statements in a program.
+
 #### Type unification
 
 Each expression must have a known type. The unpleasant option is to force the

docs/on-the-web.md

@@ -44,3 +44,11 @@ if we missed something that you think is relevant!
 | James Shubin | blog | [Mgmt Configuration Language](https://purpleidea.com/blog/2018/02/05/mgmt-configuration-language/) |
 | James Shubin | video | [Recording from CfgMgmtCamp.eu 2018](https://www.youtube.com/watch?v=NxObmwZDyrI) |
 | Jonathan Gold | blog | [Go Netlink and Select](https://jonathangold.ca/blog/go-netlink-and-select/) |
+| James Shubin | video | [Recording from DevOpsDays Montreal 2018](https://www.youtube.com/watch?v=1i38c5cooHo) |
+| James Shubin | video | [Recording from FOSDEM Minimalistic Languages Devroom 2019](https://video.fosdem.org/2019/K.4.201/mgmtconfig.webm) |
+| James Shubin | video | [Recording from FOSDEM Infra Management Devroom 2019](https://video.fosdem.org/2019/UB2.252A/mgmt.webm) |
+| James Shubin | video | [Recording from FOSDEM Graph Processing Devroom 2019](https://video.fosdem.org/2019/H.1308/graph_mgmt_config.webm) |
+| James Shubin | video | [Recording from FOSDEM Virtualization Devroom 2019](https://video.fosdem.org/2019/H.2213/vai_real_time_virtualization_automation.webm) |
+| James Shubin | video | [Recording from FOSDEM Containers Devroom 2019](https://video.fosdem.org/2019/UA2.114/containers_mgmt.webm) |
+| James Shubin | video | [Recording from FOSDEM Monitoring Devroom 2019](https://video.fosdem.org/2019/UB2.252A/real_time_merging_of_config_management_and_monitoring.webm) |
+| James Shubin | blog | [Mgmt Configuration Language: Class and Include](https://purpleidea.com/blog/2019/07/26/class-and-include-in-mgmt/) |

docs/puppet-guide.md

@@ -173,7 +173,7 @@ useful when you are in the process of replacing Puppet with mgmt. You
 can translate your custom modules into mgmt's language one by one,
 and let mgmt run the current mix.
 
-Instead of the usual `--puppet`, `--puppet-conf`, and `--lang` for mcl,
+Instead of the usual `--puppet-conf` flag and argv for `puppet` and `mcl` input,
 you need to use alternative flags to make this work:
 
 * `--lp-lang` to specify the mcl input

docs/quick-start-guide.md

@@ -2,65 +2,108 @@
 
 ## Introduction
 
-This guide is intended for developers. Once `mgmt` is minimally viable, we'll
-publish a quick start guide for users too. If you're brand new to `mgmt`, it's
-probably a good idea to start by reading the
-[introductory article](https://purpleidea.com/blog/2016/01/18/next-generation-configuration-mgmt/)
-or to watch an [introductory video](https://www.youtube.com/watch?v=LkEtBVLfygE&html5=1).
-Once you're familiar with the general idea, please start hacking...
+This guide is intended for users and developers. If you're brand new to `mgmt`,
+it's probably a good idea to start by reading an
+[introductory article about the engine](https://purpleidea.com/blog/2016/01/18/next-generation-configuration-mgmt/)
+and an [introductory article about the language](https://purpleidea.com/blog/2018/02/05/mgmt-configuration-language/).
+[There are other articles and videos available](on-the-web.md) if you'd like to
+learn more or prefer different formats. Once you're familiar with the general
+idea, or if you prefer a hands-on approach, please start hacking...
 
-## Quick start
+## Getting mgmt
 
-### Installing golang
+You can either build `mgmt` from source, or you can download a pre-built
+release. There are also some distro repositories available, but they may not be
+up to date. A pre-built release is the fastest option if there's one that's
+available for your platform. If you are developing or testing a new patch to
+`mgmt`, or there is not a release available for your platform, then you'll have
+to build your own.
 
-* You need golang version 1.10 or greater installed.
+### Downloading a pre-built release:
+
+The latest releases can be found [here](https://github.com/purpleidea/mgmt/releases/).
+An alternate mirror is available [here](https://dl.fedoraproject.org/pub/alt/purpleidea/mgmt/releases/).
+
+Make sure to verify the signatures of all packages before you use them. The
+signing key can be downloaded from [https://purpleidea.com/contact/#pgp-key](https://purpleidea.com/contact/#pgp-key)
+to verify the release.
+
+If you've decided to install a pre-build release, you can skip to the
+[Running mgmt](#running-mgmt) section below!
+
+### Building a release:
+
+You'll need some dependencies, including `golang`, and some associated tools.
+
+#### Installing golang
+
+* You need golang version 1.11 or greater installed.
 	* To install on rpm style systems: `sudo dnf install golang`
 	* To install on apt style systems: `sudo apt install golang`
 	* To install on macOS systems install [Homebrew](https://brew.sh)
 	and run: `brew install go`
 * You can run `go version` to check the golang version.
-* If your distro is tool old, you may need to [download](https://golang.org/dl/)
+* If your distro is too old, you may need to [download](https://golang.org/dl/)
 a newer golang version.
 
-### Setting up golang
+#### Setting up golang
 
-* If you do not have a GOPATH yet, create one and export it:
+* You can skip this step, as your installation will default to using `~/go/`,
+but if you do not have a `GOPATH` yet and want one in a custom location, create
+one and export it:
 
-```
+```shell
 mkdir $HOME/gopath
 export GOPATH=$HOME/gopath
 ```
 
 * You might also want to add the GOPATH to your `~/.bashrc` or `~/.profile`.
-* For more information you can read the [GOPATH documentation](https://golang.org/cmd/go/#hdr-GOPATH_environment_variable).
+* For more information you can read the
+[GOPATH documentation](https://golang.org/cmd/go/#hdr-GOPATH_environment_variable).
 
-### Getting the mgmt code and dependencies
+#### Getting the mgmt code and associated dependencies
 
-* Download the `mgmt` code into the GOPATH, and switch to that directory:
+* Download the `mgmt` code into the `GOPATH`, and switch to that directory:
 
-```
-mkdir -p $GOPATH/src/github.com/purpleidea/
-cd $GOPATH/src/github.com/purpleidea/
+```shell
+[ -z "$GOPATH" ] && mkdir ~/go/ || mkdir -p $GOPATH/src/github.com/purpleidea/
+cd $GOPATH/src/github.com/purpleidea/ || cd ~/go/
 git clone --recursive https://github.com/purpleidea/mgmt/
-cd $GOPATH/src/github.com/purpleidea/mgmt
+cd $GOPATH/src/github.com/purpleidea/mgmt/ || cd ~/go/src/github.com/purpleidea/mgmt/
 ```
 
-* Add $GOPATH/bin to $PATH
+* Add `$GOPATH/bin` to `$PATH`
 
-```
+```shell
 export PATH=$PATH:$GOPATH/bin
 ```
 
 * Run `make deps` to install system and golang dependencies. Take a look at
-`misc/make-deps.sh` for details.
-* Run `make build` to get a freshly built `mgmt` binary.
+`misc/make-deps.sh` if you want to see the details of what it does.
 
-### Running mgmt
+#### Building mgmt
 
-* Run `time ./mgmt run --tmp-prefix lang --lang examples/lang/hello0.mcl` to try
-out a very simple example!
+* Now run `make` to get a freshly built `mgmt` binary. If this succeeds, you can
+proceed to the [Running mgmt](#running-mgmt) section below!
+
+### Installing a distro release
+
+Installation of `mgmt` from distribution packages currently needs improvement.
+They are not always up-to-date with git master and as such are not recommended.
+At the moment we have:
+* [COPR](https://copr.fedoraproject.org/coprs/purpleidea/mgmt/) (currently dead)
+* [Arch](https://aur.archlinux.org/packages/mgmt/) (currently stale)
+
+Please contribute more and help improve these! We'd especially like to see a
+Debian package!
+
+## Running mgmt
+
+* Run `mgmt run --tmp-prefix lang examples/lang/hello0.mcl` to try out a very
+simple example! If you built it from source, you'll need to use `./mgmt` from
+the project directory.
 * Look in that example file that you ran to see if you can figure out what it
-did!
+did! You can press `^C` to exit `mgmt`.
 * Have fun hacking on our future technology and get involved to shape the
 project!
 
@@ -68,118 +111,3 @@ project!
 
 Please look in the [examples/lang/](../examples/lang/) folder for some more
 examples!
-
-## Vagrant
-
-If you would like to avoid doing the above steps manually, we have prepared a
-[Vagrant](https://www.vagrantup.com/) environment for your convenience. From the
-project directory, run a `vagrant up`, and then a `vagrant status`. From there,
-you can `vagrant ssh` into the `mgmt` machine. The MOTD will explain the rest.
-
-## Using Docker
-
-Alternatively, you can check out the [docker-guide](docs/docker-guide.md) in
-order to develop or deploy using docker.
-
-## Information about dependencies
-
-Software projects have a few different kinds of dependencies. There are _build_
-dependencies, _runtime_ dependencies, and additionally, a few extra dependencies
-required for running the _test_ suite.
-
-### Build
-
-* `golang` 1.10 or higher (required, available in some distros and distributed
-as a binary officially by [golang.org](https://golang.org/dl/))
-
-### Runtime
-
-A relatively modern GNU/Linux system should be able to run `mgmt` without any
-problems. Since `mgmt` runs as a single statically compiled binary, all of the
-library dependencies are included. It is expected, that certain advanced
-resources require host specific facilities to work. These requirements are
-listed below:
-
-| Resource | Dependency        | Version                     | Check version with                                        |
-|----------|-------------------|-----------------------------|-----------------------------------------------------------|
-| augeas   | augeas-devel      | `augeas 1.6` or greater     | `dnf info augeas-devel` or `apt-cache show libaugeas-dev` |
-| file     | inotify           | `Linux 2.6.27` or greater   | `uname -a`                                                |
-| hostname | systemd-hostnamed | `systemd 25` or greater     | `systemctl --version`                                     |
-| nspawn   | systemd-nspawn    | `systemd ???` or greater    | `systemctl --version`                                     |
-| pkg      | packagekitd       | `packagekit 1.x` or greater | `pkcon --version`                                         |
-| svc      | systemd           | `systemd ???` or greater    | `systemctl --version`                                     |
-| virt     | libvirt-devel     | `libvirt 1.2.0` or greater  | `dnf info libvirt-devel` or `apt-cache show libvirt-dev`  |
-| virt     | libvirtd          | `libvirt 1.2.0` or greater  | `libvirtd --version`                                      |
-
-For building a visual representation of the graph, `graphviz` is required.
-
-To build `mgmt` without augeas support please run:
-`GOTAGS='noaugeas' make build`
-
-To build `mgmt` without libvirt support please run:
-`GOTAGS='novirt' make build`
-
-To build `mgmt` without docker support please run:
-`GOTAGS='nodocker' make build`
-
-To build `mgmt` without augeas, libvirt or docker support please run:
-`GOTAGS='noaugeas novirt nodocker' make build`
-
-## Binary Package Installation
-
-Installation of `mgmt` from distribution packages currently needs improvement.
-They are not always up-to-date with git master and as such are not recommended.
-At the moment we have:
-* [COPR](https://copr.fedoraproject.org/coprs/purpleidea/mgmt/)
-* [Arch](https://aur.archlinux.org/packages/mgmt/)
-
-Please contribute more! We'd especially like to see a Debian package!
-
-## OSX/macOS/Darwin development
-
-Developing and running `mgmt` on macOS is currently not supported (but not
-discouraged either). Meaning it might work but in the case it doesn't you would
-have to provide your own patches to fix problems (the project maintainer and
-community are glad to assist where needed).
-
-There are currently some issues that make `mgmt` less suitable to run for provisioning
-macOS. But as a client to provision remote servers it should run fine.
-
-Since the primary supported systems are Linux and these are the environments
-tested for it is wise to run these suites during macOS development as well. To
-ease this Docker can be leveraged ([Docker for Mac](https://docs.docker.com/docker-for-mac/)).
-
-Before running any of the commands below create the development Docker image:
-
-```
-docker/scripts/build-development
-```
-
-This image requires updating every time dependencies (`make-deps.sh`) change.
-
-Then to run the test suite:
-
-```
-docker run --rm -ti \
-	-v $PWD:/go/src/github.com/purpleidea/mgmt/ \
-	-w /go/src/github.com/purpleidea/mgmt/ \
-	purpleidea/mgmt:development \
-	make test
-```
-
-For convenience this command is wrapped in `docker/scripts/exec-development`.
-
-Basically any command can be executed this way. Because the repository source is
-mounted into the Docker container invocation will be quick and allow rapid
-testing, example:
-
-```
-docker/scripts/exec-development test/test-shell.sh load0.sh
-```
-
-Other examples:
-
-```
-docker/scripts/exec-development make build
-docker/scripts/exec-development ./mgmt run --tmp-prefix lang --lang examples/lang/load0.mcl
-```

docs/resources.md

@@ -69,8 +69,8 @@ identified by a trailing slash in their path name. File have no such slash.
 It has the following properties:
 
 * `path`: absolute file path (directories have a trailing slash here)
+* `state`: either `exists`, `absent`, or undefined
 * `content`: raw file content
-* `state`: either `exists` (the default value) or `absent`
 * `mode`: octal unix file permissions
 * `owner`: username or uid for the file owner
 * `group`: group name or gid for the file group
@@ -79,6 +79,16 @@ It has the following properties:
 
 The path property specifies the file or directory that we are managing.
 
+### State
+
+The state property describes the action we'd like to apply for the resource. The
+possible values are: `exists` and `absent`. If you do not specify either of
+these, it is undefined. Without specifying this value as `exists`, another param
+cannot cause a file to get implicitly created. When specifying this value as
+`absent`, you should not specify any other params that would normally change the
+file. For example, if you specify `content` and this param is `absent`, then you
+will get an engine validation error.
+
 ### Content
 
 The content property is a string that specifies the desired file contents.
@@ -88,11 +98,6 @@ The content property is a string that specifies the desired file contents.
 The source property points to a source file or directory path that we wish to
 copy over and use as the desired contents for our resource.
 
-### State
-
-The state property describes the action we'd like to apply for the resource. The
-possible values are: `exists` and `absent`.
-
 ### Recurse
 
 The recurse property limits whether file resource operations should recurse into

engine/cmp.go

@@ -152,6 +152,18 @@ func ResCmp(r1, r2 Res) error {
 		}
 	}
 
+	// compare meta params for resources with reversible traits
+	r1v, ok1 := r1.(ReversibleRes)
+	r2v, ok2 := r2.(ReversibleRes)
+	if ok1 != ok2 {
+		return fmt.Errorf("reversible differs") // they must be different (optional)
+	}
+	if ok1 && ok2 {
+		if r1v.ReversibleMeta().Cmp(r2v.ReversibleMeta()) != nil {
+			return fmt.Errorf("reversible differs")
+		}
+	}
+
 	return nil
 }
 
@@ -280,6 +292,18 @@ func AdaptCmp(r1, r2 CompatibleRes) error {
 		}
 	}
 
+	// compare meta params for resources with reversible traits
+	r1v, ok1 := r1.(ReversibleRes)
+	r2v, ok2 := r2.(ReversibleRes)
+	if ok1 != ok2 {
+		return fmt.Errorf("reversible differs") // they must be different (optional)
+	}
+	if ok1 && ok2 {
+		if r1v.ReversibleMeta().Cmp(r2v.ReversibleMeta()) != nil {
+			return fmt.Errorf("reversible differs")
+		}
+	}
+
 	return nil
 }
 

engine/copy.go

@@ -106,6 +106,16 @@ func ResCopy(r CopyableRes) (CopyableRes, error) {
 		}
 	}
 
+	// copy meta params for resources with reversible traits
+	if x, ok := r.(ReversibleRes); ok {
+		dst, ok := res.(ReversibleRes)
+		if !ok {
+			// programming error
+			panic("reversible interfaces are illogical")
+		}
+		dst.SetReversibleMeta(x.ReversibleMeta()) // no need to copy atm
+	}
+
 	return res, nil
 }
 

engine/graph/autoedge/autoedge.go

@@ -89,6 +89,9 @@ func AutoEdge(graph *pgraph.Graph, debug bool, logf func(format string, v ...int
 			}
 		}
 	}
+
+	// It would be great to ensure we didn't add any loops here, but instead
+	// of checking now, we'll move the check into the main loop.
 	return nil
 }
 

engine/graph/autogroup/autogroup.go

@@ -66,5 +66,8 @@ func AutoGroup(ag engine.AutoGrouper, g *pgraph.Graph, debug bool, logf func(for
 		}
 	}
 
+	// It would be great to ensure we didn't add any loops here, but instead
+	// of checking now, we'll move the check into the main loop.
+
 	return nil
 }

engine/graph/autogroup/util.go

@@ -18,6 +18,9 @@
 package autogroup
 
 import (
+	"fmt"
+
+	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util/errwrap"
 )
@@ -112,8 +115,17 @@ func VertexMerge(g *pgraph.Graph, v1, v2 pgraph.Vertex, vertexMergeFn func(pgrap
 			// note: This branch isn't used if the vertexMergeFn
 			// decides to just merge logically on its own instead
 			// of actually returning something that we then merge.
-			v1 = v // TODO: ineffassign?
+			v1 = v // XXX: ineffassign?
 			//*v1 = *v
+
+			// Ensure that everything still validates. (For safety!)
+			r, ok := v1.(engine.Res) // TODO: v ?
+			if !ok {
+				return fmt.Errorf("not a Res")
+			}
+			if err := engine.Validate(r); err != nil {
+				return errwrap.Wrapf(err, "the Res did not Validate")
+			}
 		}
 	}
 	g.DeleteVertex(v2) // remove grouped vertex

engine/graph/engine.go

@@ -25,11 +25,18 @@ import (
 
 	"github.com/purpleidea/mgmt/converger"
 	"github.com/purpleidea/mgmt/engine"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util/errwrap"
 	"github.com/purpleidea/mgmt/util/semaphore"
 )
 
+const (
+	// StateDir is the name of the sub directory where all the local
+	// resource state is stored.
+	StateDir = "state"
+)
+
 // Engine encapsulates a generic graph and manages its operations.
 type Engine struct {
 	Program  string
@@ -48,6 +55,7 @@ type Engine struct {
 	nextGraph *pgraph.Graph
 	state     map[pgraph.Vertex]*State
 	waits     map[pgraph.Vertex]*sync.WaitGroup // wg for the Worker func
+	wlock     *sync.Mutex                       // lock around waits map
 
 	slock *sync.Mutex // semaphore lock
 	semas map[string]*semaphore.Semaphore
@@ -83,6 +91,7 @@ func (obj *Engine) Init() error {
 
 	obj.state = make(map[pgraph.Vertex]*State)
 	obj.waits = make(map[pgraph.Vertex]*sync.WaitGroup)
+	obj.wlock = &sync.Mutex{}
 
 	obj.slock = &sync.Mutex{}
 	obj.semas = make(map[string]*semaphore.Semaphore)
@@ -172,9 +181,9 @@ func (obj *Engine) Commit() error {
 			return errwrap.Wrapf(err, "the Res did not Validate")
 		}
 
-		// FIXME: is res.Name() sufficiently unique to use as a UID here?
-		pathUID := fmt.Sprintf("%s-%s", res.Kind(), res.Name())
-		statePrefix := fmt.Sprintf("%s/", path.Join(obj.Prefix, "state", pathUID))
+		pathUID := engineUtil.ResPathUID(res)
+		statePrefix := fmt.Sprintf("%s/", path.Join(obj.statePrefix(), pathUID))
+
 		// don't create this unless it *will* be used
 		//if err := os.MkdirAll(statePrefix, 0770); err != nil {
 		//	return errwrap.Wrapf(err, "can't create state prefix")
@@ -204,10 +213,19 @@ func (obj *Engine) Commit() error {
 		fn := func() error {
 			// start the Worker
 			obj.wg.Add(1)
+			obj.wlock.Lock()
 			obj.waits[vertex].Add(1)
+			obj.wlock.Unlock()
 			go func(v pgraph.Vertex) {
 				defer obj.wg.Done()
-				defer obj.waits[v].Done()
+				defer func() {
+					// we need this lock, because this go
+					// routine could run when the next fn
+					// function above here is running...
+					obj.wlock.Lock()
+					obj.waits[v].Done()
+					obj.wlock.Unlock()
+				}()
 
 				obj.Logf("Worker(%s)", v)
 				// contains the Watch and CheckApply loops
@@ -405,3 +423,8 @@ func (obj *Engine) Close() error {
 func (obj *Engine) Graph() *pgraph.Graph {
 	return obj.graph
 }
+
+// statePrefix returns the dir where all the resource state is stored locally.
+func (obj *Engine) statePrefix() string {
+	return fmt.Sprintf("%s/", path.Join(obj.Prefix, StateDir))
+}

engine/graph/reverse.go

@@ -0,0 +1,295 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graph
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"sort"
+
+	"github.com/purpleidea/mgmt/engine"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+const (
+	// ReverseFile is the file name in the resource state dir where any
+	// reversal information is stored.
+	ReverseFile = "reverse"
+
+	// ReversePerm is the permissions mode used to create the ReverseFile.
+	ReversePerm = 0600
+)
+
+// Reversals adds the reversals onto the loaded graph. This should happen last,
+// and before Commit.
+func (obj *Engine) Reversals() error {
+	if obj.nextGraph == nil {
+		return fmt.Errorf("there is no active graph to add reversals to")
+	}
+
+	// Initially get all of the reversals to seek out all possible errors.
+	// XXX: The engine needs to know where data might have been stored if we
+	// XXX: want to potentially allow alternate read/write paths, like etcd.
+	// XXX: In this scenario, we'd have to store a token somewhere to let us
+	// XXX: know to look elsewhere for the special ReversalList read method.
+	data, err := obj.ReversalList() // (map[string]string, error)
+	if err != nil {
+		return errwrap.Wrapf(err, "the reversals had errors")
+	}
+
+	if len(data) == 0 {
+		return nil // end early
+	}
+
+	resMatch := func(r1, r2 engine.Res) bool { // simple match on UID only!
+		if r1.Kind() != r2.Kind() {
+			return false
+		}
+		if r1.Name() != r2.Name() {
+			return false
+		}
+		return true
+	}
+	resInList := func(needle engine.Res, haystack []engine.Res) bool {
+		for _, res := range haystack {
+			if resMatch(needle, res) {
+				return true
+			}
+		}
+		return false
+	}
+
+	if obj.Debug {
+		obj.Logf("decoding %d reversals...", len(data))
+	}
+	resources := []engine.Res{}
+
+	// do this in a sorted order so that it errors deterministically
+	sorted := []string{}
+	for key := range data {
+		sorted = append(sorted, key)
+	}
+	sort.Strings(sorted)
+	for _, key := range sorted {
+		val := data[key]
+		// XXX: replace this ResToB64 method with one that stores it in
+		// a human readable format, in case someone wants to hack and
+		// edit it manually.
+		// XXX: we probably want this to be YAML, it works with the diff
+		// too...
+		r, err := engineUtil.B64ToRes(val)
+		if err != nil {
+			return errwrap.Wrapf(err, "error decoding res with UID: `%s`", key)
+		}
+
+		res, ok := r.(engine.ReversibleRes)
+		if !ok {
+			// this requirement is here to keep things simpler...
+			return errwrap.Wrapf(err, "decoded res with UID: `%s` was not reversible", key)
+		}
+
+		matchFn := func(vertex pgraph.Vertex) (bool, error) {
+			r, ok := vertex.(engine.Res)
+			if !ok {
+				return false, fmt.Errorf("not a Res")
+			}
+			if !resMatch(r, res) {
+				return false, nil
+			}
+			return true, nil
+		}
+
+		// FIXME: not efficient, we could build a cache-map first
+		vertex, err := obj.nextGraph.VertexMatchFn(matchFn) // (Vertex, error)
+		if err != nil {
+			return errwrap.Wrapf(err, "error searching graph for match")
+		}
+		if vertex != nil { // found one!
+			continue // it doesn't need reversing yet
+		}
+
+		// TODO: check for (incompatible?) duplicates instead
+		if resInList(res, resources) { // we've already got this one...
+			continue
+		}
+
+		// We set this in two different places to be safe. It ensures
+		// that we erase the reversal state file after we've used it.
+		res.ReversibleMeta().Reversal = true // set this for later...
+
+		resources = append(resources, res)
+	}
+
+	if len(resources) == 0 {
+		return nil // end early
+	}
+
+	// Now that we've passed the chance of any errors, we modify the graph.
+	obj.Logf("adding %d reversals...", len(resources))
+	for _, res := range resources {
+		obj.nextGraph.AddVertex(res)
+	}
+	// TODO: Do we want a way for stored reversals to add edges too?
+
+	// It would be great to ensure we didn't add any loops here, but instead
+	// of checking now, we'll move the check into the main loop.
+	return nil
+}
+
+// ReversalList returns all the available pending reversal data on this host. It
+// can then be decoded by whatever method is appropriate for.
+func (obj *Engine) ReversalList() (map[string]string, error) {
+	result := make(map[string]string) // some key to contents
+
+	dir := obj.statePrefix() // loop through this dir...
+	files, err := ioutil.ReadDir(dir)
+	if err != nil && !os.IsNotExist(err) {
+		return nil, errwrap.Wrapf(err, "error reading list of state dirs")
+	} else if err != nil {
+		return result, nil // nothing found, no state dir exists yet
+	}
+
+	for _, x := range files {
+		key := x.Name() // some uid for the resource
+		file := path.Join(dir, key, ReverseFile)
+		content, err := ioutil.ReadFile(file)
+		if err != nil && !os.IsNotExist(err) {
+			return nil, errwrap.Wrapf(err, "could not read reverse file: %s", file)
+		} else if err != nil {
+			continue // file does not exist, skip
+		}
+
+		// file exists!
+		str := string(content)
+		result[key] = str // save
+	}
+
+	return result, nil
+}
+
+// ReversalInit performs the reversal initialization steps if necessary for this
+// resource.
+func (obj *State) ReversalInit() error {
+	res, ok := obj.Vertex.(engine.ReversibleRes)
+	if !ok {
+		return nil // nothing to do
+	}
+
+	if res.ReversibleMeta().Disabled {
+		return nil // nothing to do, reversal isn't enabled
+	}
+
+	// If the reversal is enabled, but we are the result of a previous
+	// reversal, then this will overwrite that older reversal request, and
+	// our resource should be designed to deal with that. This happens if we
+	// return a reversible resource as the reverse of a resource that was
+	// reversed. It's probably fairly rare.
+	if res.ReversibleMeta().Reversal {
+		obj.Logf("triangle reversal") // warn!
+	}
+
+	r, err := res.Reversed()
+	if err != nil {
+		return errwrap.Wrapf(err, "could not reverse: %s", res.String())
+	}
+	if r == nil {
+		return nil // this can't be reversed, or isn't implemented here
+	}
+
+	// We set this in two different places to be safe. It ensures that we
+	// erase the reversal state file after we've used it.
+	r.ReversibleMeta().Reversal = true // set this for later...
+
+	// XXX: replace this ResToB64 method with one that stores it in a human
+	// readable format, in case someone wants to hack and edit it manually.
+	// XXX: we probably want this to be YAML, it works with the diff too...
+	str, err := engineUtil.ResToB64(r)
+	if err != nil {
+		return errwrap.Wrapf(err, "could not encode: %s", res.String())
+	}
+
+	// TODO: put this method on traits.Reversible as part of the interface?
+	return obj.ReversalWrite(str, res.ReversibleMeta().Overwrite) // Store!
+}
+
+// ReversalClose performs the reversal shutdown steps if necessary for this
+// resource.
+func (obj *State) ReversalClose() error {
+	res, ok := obj.Vertex.(engine.ReversibleRes)
+	if !ok {
+		return nil // nothing to do
+	}
+
+	// Don't check res.ReversibleMeta().Disabled because we're removing the
+	// previous one. That value only applies if we're doing a new reversal.
+
+	if !res.ReversibleMeta().Reversal {
+		return nil // nothing to erase, we're not a reversal resource
+	}
+
+	if !obj.isStateOK { // did we successfully reverse?
+		obj.Logf("did not complete reversal") // warn
+		return nil
+	}
+
+	// TODO: put this method on traits.Reversible as part of the interface?
+	return obj.ReversalDelete() // Erase our reversal instructions.
+}
+
+// ReversalWrite stores the reversal state information for this resource.
+func (obj *State) ReversalWrite(str string, overwrite bool) error {
+	dir, err := obj.varDir("") // private version
+	if err != nil {
+		return errwrap.Wrapf(err, "could not get VarDir for reverse")
+	}
+	file := path.Join(dir, ReverseFile) // return a unique file
+
+	content, err := ioutil.ReadFile(file)
+	if err != nil && !os.IsNotExist(err) {
+		return errwrap.Wrapf(err, "could not read reverse file: %s", file)
+	}
+
+	// file exists and we shouldn't overwrite if different
+	if err == nil && !overwrite {
+		// compare to existing file
+		oldStr := string(content)
+		if str != oldStr {
+			obj.Logf("existing, pending, reversible resource exists")
+			//obj.Logf("diff:")
+			//obj.Logf("") // TODO: print the diff w/o and secret values
+			return fmt.Errorf("existing, pending, reversible resource exists")
+		}
+	}
+
+	return ioutil.WriteFile(file, []byte(str), ReversePerm)
+}
+
+// ReversalDelete removes the reversal state information for this resource.
+func (obj *State) ReversalDelete() error {
+	dir, err := obj.varDir("") // private version
+	if err != nil {
+		return errwrap.Wrapf(err, "could not get VarDir for reverse")
+	}
+	file := path.Join(dir, ReverseFile) // return a unique file
+
+	return errwrap.Wrapf(os.Remove(file), "could not remove reverse state file")
+}

engine/graph/state.go

@@ -203,6 +203,12 @@ func (obj *State) Init() error {
 	if obj.Debug {
 		obj.Logf("Init(%s)", res)
 	}
+
+	// write the reverse request to the disk...
+	if err := obj.ReversalInit(); err != nil {
+		return err // TODO: test this code path...
+	}
+
 	err := res.Init(obj.init)
 	if obj.Debug {
 		obj.Logf("Init(%s): Return(%+v)", res, err)
@@ -236,12 +242,23 @@ func (obj *State) Close() error {
 	if obj.Debug {
 		obj.Logf("Close(%s)", res)
 	}
-	err := res.Close()
-	if obj.Debug {
-		obj.Logf("Close(%s): Return(%+v)", res, err)
+
+	var reverr error
+	// clear the reverse request from the disk...
+	if err := obj.ReversalClose(); err != nil {
+		// TODO: test this code path...
+		// TODO: should this be an error or a warning?
+		reverr = err
 	}
 
-	return err
+	reterr := res.Close()
+	if obj.Debug {
+		obj.Logf("Close(%s): Return(%+v)", res, reterr)
+	}
+
+	reterr = errwrap.Append(reterr, reverr)
+
+	return reterr
 }
 
 // Poke sends a notification on the poke channel. This channel is used to notify

engine/resources/aws_ec2.go

@@ -751,45 +751,37 @@ func (obj *AwsEc2Res) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *AwsEc2Res) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *AwsEc2Res) Compare(r engine.Res) bool {
 	// we can only compare AwsEc2Res to others of the same resource kind
 	res, ok := r.(*AwsEc2Res)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.State != res.State {
-		return false
+		return fmt.Errorf("the State differs")
 	}
 	if obj.Region != res.Region {
-		return false
+		return fmt.Errorf("the Region differs")
 	}
 	if obj.Type != res.Type {
-		return false
+		return fmt.Errorf("the Type differs")
 	}
 	if obj.ImageID != res.ImageID {
-		return false
+		return fmt.Errorf("the ImageID differs")
 	}
 	if obj.WatchEndpoint != res.WatchEndpoint {
-		return false
+		return fmt.Errorf("the WatchEndpoint differs")
 	}
 	if obj.WatchListenAddr != res.WatchListenAddr {
-		return false
+		return fmt.Errorf("the WatchListenAddr differs")
 	}
 	if obj.ErrorOnMalformedPost != res.ErrorOnMalformedPost {
-		return false
+		return fmt.Errorf("the ErrorOnMalformedPost differs")
 	}
 	if obj.UserData != res.UserData {
-		return false
+		return fmt.Errorf("the UserData differs")
 	}
-	return true
+	return nil
 }
 
 func (obj *AwsEc2Res) prependName() string {
@@ -1025,7 +1017,7 @@ func (obj *AwsEc2Res) snsMakeTopic() (string, error) {
 	}
 	obj.init.Logf("Created SNS Topic")
 	if topic.TopicArn == nil {
-		return "", fmt.Errorf("TopicArn is nil")
+		return "", fmt.Errorf("the TopicArn is nil")
 	}
 	return *topic.TopicArn, nil
 }

engine/resources/file.go

@@ -43,6 +43,18 @@ func init() {
 	engine.RegisterResource("file", func() engine.Res { return &FileRes{} })
 }
 
+const (
+	// FileStateExists is the string that represents that the file should be
+	// present.
+	FileStateExists = "exists"
+	// FileStateAbsent is the string that represents that the file should
+	// not exist.
+	FileStateAbsent = "absent"
+	// FileStateUndefined means the file state has not been specified.
+	// TODO: consider moving to *string and express this state as a nil.
+	FileStateUndefined = ""
+)
+
 // FileRes is a file and directory resource. Dirs are defined by names ending
 // in a slash.
 type FileRes struct {
@@ -50,6 +62,7 @@ type FileRes struct {
 	traits.Edgeable
 	//traits.Groupable // TODO: implement this
 	traits.Recvable
+	traits.Reversible
 
 	init *engine.Init
 
@@ -60,19 +73,29 @@ type FileRes struct {
 	Dirname  string `lang:"dirname" yaml:"dirname"`   // override the path dirname
 	Basename string `lang:"basename" yaml:"basename"` // override the path basename
 
+	// State specifies the desired state of the file. It can be either
+	// `exists` or `absent`. If you do not specify this, we will not be able
+	// to create or remove a file if it might be logical for another
+	// param to require that. Instead it will error. This means that this
+	// field is not implied by specifying some content or a mode.
+	State string `lang:"state" yaml:"state"`
+
 	// Content specifies the file contents to use. If this is nil, they are
 	// left undefined. It cannot be combined with Source.
 	Content *string `lang:"content" yaml:"content"`
 	// Source specifies the source contents for the file resource. It cannot
 	// be combined with the Content parameter.
 	Source string `lang:"source" yaml:"source"`
-	// State specifies the desired state of the file. It can be either
-	// `exists` or `absent`. If you do not specify this, it will be
-	// undefined, and determined based on the other parameters.
-	State string `lang:"state" yaml:"state"`
 
-	Owner   string `lang:"owner" yaml:"owner"`
-	Group   string `lang:"group" yaml:"group"`
+	// Owner specifies the file owner. You can specify either the string
+	// name, or a string representation of the owner integer uid.
+	Owner string `lang:"owner" yaml:"owner"`
+	// Group specifies the file group. You can specify either the string
+	// name, or a string representation of the group integer gid.
+	Group string `lang:"group" yaml:"group"`
+	// Mode is the mode of the file as a string representation of the octal
+	// form.
+	// TODO: add symbolic representations
 	Mode    string `lang:"mode" yaml:"mode"`
 	Recurse bool   `lang:"recurse" yaml:"recurse"`
 	Force   bool   `lang:"force" yaml:"force"`
@@ -81,96 +104,6 @@ type FileRes struct {
 	recWatcher *recwatch.RecWatcher
 }
 
-// Default returns some sensible defaults for this resource.
-func (obj *FileRes) Default() engine.Res {
-	return &FileRes{
-		State: "exists",
-	}
-}
-
-// Validate reports any problems with the struct definition.
-func (obj *FileRes) Validate() error {
-	if obj.getPath() == "" {
-		return fmt.Errorf("path is empty")
-	}
-
-	if obj.Dirname != "" && !strings.HasSuffix(obj.Dirname, "/") {
-		return fmt.Errorf("dirname must end with a slash")
-	}
-
-	if strings.HasPrefix(obj.Basename, "/") {
-		return fmt.Errorf("basename must not start with a slash")
-	}
-
-	if !strings.HasPrefix(obj.getPath(), "/") {
-		return fmt.Errorf("resultant path must be absolute")
-	}
-
-	if obj.Content != nil && obj.Source != "" {
-		return fmt.Errorf("can't specify both Content and Source")
-	}
-
-	if obj.isDir() && obj.Content != nil { // makes no sense
-		return fmt.Errorf("can't specify Content when creating a Dir")
-	}
-
-	if obj.Mode != "" {
-		if _, err := obj.mode(); err != nil {
-			return err
-		}
-	}
-
-	if obj.Owner != "" || obj.Group != "" {
-		fileInfo, err := os.Stat("/") // pick root just to do this test
-		if err != nil {
-			return fmt.Errorf("can't stat root to get system information")
-		}
-		_, ok := fileInfo.Sys().(*syscall.Stat_t)
-		if !ok {
-			return fmt.Errorf("can't set Owner or Group on this platform")
-		}
-	}
-	if _, err := engineUtil.GetUID(obj.Owner); obj.Owner != "" && err != nil {
-		return err
-	}
-
-	if _, err := engineUtil.GetGID(obj.Group); obj.Group != "" && err != nil {
-		return err
-	}
-
-	// XXX: should this specify that we create an empty directory instead?
-	//if obj.Source == "" && obj.isDir() {
-	//	return fmt.Errorf("Can't specify an empty source when creating a Dir.")
-	//}
-
-	return nil
-}
-
-// mode returns the file permission specified on the graph. It doesn't handle
-// the case where the mode is not specified. The caller should check obj.Mode is
-// not empty.
-func (obj *FileRes) mode() (os.FileMode, error) {
-	m, err := strconv.ParseInt(obj.Mode, 8, 32)
-	if err != nil {
-		return os.FileMode(0), errwrap.Wrapf(err, "mode should be an octal number (%s)", obj.Mode)
-	}
-	return os.FileMode(m), nil
-}
-
-// Init runs some startup code for this resource.
-func (obj *FileRes) Init(init *engine.Init) error {
-	obj.init = init // save for later
-
-	obj.sha256sum = ""
-
-	return nil
-}
-
-// Close is run by the engine to clean up after the resource is done.
-func (obj *FileRes) Close() error {
-	return nil
-}
-
 // getPath returns the actual path to use for this resource. It computes this
 // after analysis of the Path, Dirname and Basename values. Dirs end with slash.
 // TODO: memoize the result if this seems important.
@@ -200,6 +133,115 @@ func (obj *FileRes) isDir() bool {
 	return strings.HasSuffix(obj.getPath(), "/") // dirs have trailing slashes
 }
 
+// mode returns the file permission specified on the graph. It doesn't handle
+// the case where the mode is not specified. The caller should check obj.Mode is
+// not empty.
+func (obj *FileRes) mode() (os.FileMode, error) {
+	m, err := strconv.ParseInt(obj.Mode, 8, 32)
+	if err != nil {
+		return os.FileMode(0), errwrap.Wrapf(err, "mode should be an octal number (%s)", obj.Mode)
+	}
+	return os.FileMode(m), nil
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *FileRes) Default() engine.Res {
+	return &FileRes{
+		//State: FileStateUndefined, // the default must be undefined!
+	}
+}
+
+// Validate reports any problems with the struct definition.
+func (obj *FileRes) Validate() error {
+	if obj.getPath() == "" {
+		return fmt.Errorf("path is empty")
+	}
+
+	if obj.Dirname != "" && !strings.HasSuffix(obj.Dirname, "/") {
+		return fmt.Errorf("dirname must end with a slash")
+	}
+
+	if strings.HasPrefix(obj.Basename, "/") {
+		return fmt.Errorf("basename must not start with a slash")
+	}
+
+	if !strings.HasPrefix(obj.getPath(), "/") {
+		return fmt.Errorf("resultant path must be absolute")
+	}
+
+	if obj.State != FileStateExists && obj.State != FileStateAbsent && obj.State != FileStateUndefined {
+		return fmt.Errorf("the State is invalid")
+	}
+
+	if obj.State == FileStateAbsent && obj.Content != nil {
+		return fmt.Errorf("can't specify Content for an absent file")
+	}
+
+	if obj.Content != nil && obj.Source != "" {
+		return fmt.Errorf("can't specify both Content and Source")
+	}
+
+	if obj.isDir() && obj.Content != nil { // makes no sense
+		return fmt.Errorf("can't specify Content when creating a Dir")
+	}
+
+	// TODO: should we silently ignore these errors or include them?
+	//if obj.State == FileStateAbsent && obj.Owner != "" {
+	//	return fmt.Errorf("can't specify Owner for an absent file")
+	//}
+	//if obj.State == FileStateAbsent && obj.Group != "" {
+	//	return fmt.Errorf("can't specify Group for an absent file")
+	//}
+	if obj.Owner != "" || obj.Group != "" {
+		fileInfo, err := os.Stat("/") // pick root just to do this test
+		if err != nil {
+			return fmt.Errorf("can't stat root to get system information")
+		}
+		_, ok := fileInfo.Sys().(*syscall.Stat_t)
+		if !ok {
+			return fmt.Errorf("can't set Owner or Group on this platform")
+		}
+	}
+	if _, err := engineUtil.GetUID(obj.Owner); obj.Owner != "" && err != nil {
+		return err
+	}
+
+	if _, err := engineUtil.GetGID(obj.Group); obj.Group != "" && err != nil {
+		return err
+	}
+
+	// TODO: should we silently ignore this error or include it?
+	//if obj.State == FileStateAbsent && obj.Mode != "" {
+	//	return fmt.Errorf("can't specify Mode for an absent file")
+	//}
+	if obj.Mode != "" {
+		if _, err := obj.mode(); err != nil {
+			return err
+		}
+	}
+
+	// XXX: should this specify that we create an empty directory instead?
+	//if obj.Source == "" && obj.isDir() {
+	//	return fmt.Errorf("can't specify an empty source when creating a Dir.")
+	//}
+
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *FileRes) Init(init *engine.Init) error {
+	obj.init = init // save for later
+
+	obj.sha256sum = ""
+
+	return nil
+}
+
+// Close is run by the engine to clean up after the resource is done.
+func (obj *FileRes) Close() error {
+	return nil
+}
+
 // Watch is the primary listener for this resource and it outputs events.
 // This one is a file watcher for files and directories.
 // Modify with caution, it is probably important to write some test cases first!
@@ -252,7 +294,7 @@ func (obj *FileRes) Watch() error {
 // can be a bytes Buffer struct. It can take an input sha256 hash to use instead
 // of computing the source data hash, and it returns the computed value if this
 // function reaches that stage. As usual, it respects the apply action variable,
-// and it symmetry with the main CheckApply function returns checkOK and error.
+// and has some symmetry with the main CheckApply function.
 func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sha256sum string) (string, bool, error) {
 	// TODO: does it make sense to switch dst to an io.Writer ?
 	// TODO: use obj.Force when dealing with symlinks and other file types!
@@ -289,18 +331,25 @@ func (obj *FileRes) fileCheckApply(apply bool, src io.ReadSeeker, dst string, sh
 	defer dstClose()
 	dstExists := !os.IsNotExist(err)
 
+	// Optimization: we shouldn't be making the file, it happens in
+	// stateCheckApply, but we skip doing it there in order to do it here,
+	// unless we're undefined, and then we shouldn't force it!
+	if !dstExists && obj.State == FileStateUndefined {
+		return "", false, err
+	}
+
 	dstStat, err := dstFile.Stat()
 	if err != nil && dstExists {
 		return "", false, err
 	}
 
 	if dstExists && dstStat.IsDir() { // oops, dst is a dir, and we want a file...
-		if !apply {
-			return "", false, nil
-		}
 		if !obj.Force {
 			return "", false, fmt.Errorf("can't force dir into file: %s", dst)
 		}
+		if !apply {
+			return "", false, nil
+		}
 
 		cleanDst := path.Clean(dst)
 		if cleanDst == "" || cleanDst == "/" {
@@ -390,7 +439,7 @@ func (obj *FileRes) dirCheckApply(apply bool) (bool, error) {
 	// check if the path exists and is a directory
 	fileInfo, err := os.Stat(obj.getPath())
 	if err != nil && !os.IsNotExist(err) {
-		return false, errwrap.Wrapf(err, "error checking file resource existence")
+		return false, errwrap.Wrapf(err, "stat error on file resource")
 	}
 
 	if err == nil && fileInfo.IsDir() {
@@ -503,6 +552,7 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 				relPathFile := strings.TrimSuffix(relPath, "/")
 				if _, ok := smartDst[relPathFile]; ok {
 					absCleanDst := path.Clean(absDst)
+					// TODO: can we fail this before `!apply`?
 					if !obj.Force {
 						return false, fmt.Errorf("can't force file into dir: %s", absCleanDst)
 					}
@@ -571,13 +621,13 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 			continue
 		}
 		_ = absSrc
-		//obj.init.Logf("syncCheckApply: Recurse rm: %s -> %s", absSrc, absDst)
+		//obj.init.Logf("syncCheckApply: recurse rm: %s -> %s", absSrc, absDst)
 		//if c, err := obj.syncCheckApply(apply, absSrc, absDst); err != nil {
-		//	return false, errwrap.Wrapf(err, "syncCheckApply: Recurse rm failed")
+		//	return false, errwrap.Wrapf(err, "syncCheckApply: recurse rm failed")
 		//} else if !c { // don't let subsequent passes make this true
 		//	checkOK = false
 		//}
-		//obj.init.Logf("syncCheckApply: Removing: %s", absCleanDst)
+		//obj.init.Logf("syncCheckApply: removing: %s", absCleanDst)
 		//if apply { // safety
 		//	if err := os.Remove(absCleanDst); err != nil {
 		//		return false, err
@@ -589,9 +639,10 @@ func (obj *FileRes) syncCheckApply(apply bool, src, dst string) (bool, error) {
 	return checkOK, nil
 }
 
-// state performs a CheckApply of the file state to create an empty file.
+// stateCheckApply performs a CheckApply of the file state to create or remove
+// an empty file or directory.
 func (obj *FileRes) stateCheckApply(apply bool) (bool, error) {
-	if obj.State == "" { // state is not specified
+	if obj.State == FileStateUndefined { // state is not specified
 		return true, nil
 	}
 
@@ -601,11 +652,11 @@ func (obj *FileRes) stateCheckApply(apply bool) (bool, error) {
 		return false, errwrap.Wrapf(err, "could not stat file")
 	}
 
-	if obj.State == "absent" && os.IsNotExist(err) {
+	if obj.State == FileStateAbsent && os.IsNotExist(err) {
 		return true, nil
 	}
 
-	if obj.State == "exists" && err == nil {
+	if obj.State == FileStateExists && err == nil {
 		return true, nil
 	}
 
@@ -614,153 +665,107 @@ func (obj *FileRes) stateCheckApply(apply bool) (bool, error) {
 		return false, nil
 	}
 
-	if obj.State == "absent" {
-		return false, nil // defer the work to contentCheckApply
-	}
-
-	if obj.Content == nil && !obj.isDir() {
-		// Create an empty file to ensure one exists. Don't O_TRUNC it,
-		// in case one is magically created right after our exists test.
-		// The chmod used is what is used by the os.Create function.
-		// TODO: is using O_EXCL okay?
-		f, err := os.OpenFile(obj.getPath(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
-		if err != nil {
-			return false, errwrap.Wrapf(err, "problem creating empty file")
+	if obj.State == FileStateAbsent { // remove
+		p := obj.getPath()
+		if p == "" {
+			// programming error?
+			return false, fmt.Errorf("can't remove empty path") // safety
 		}
-		if err := f.Close(); err != nil {
-			return false, errwrap.Wrapf(err, "problem closing empty file")
-		}
-	}
-
-	return false, nil // defer the Content != nil and isDir work to later...
-}
-
-// contentCheckApply performs a CheckApply for the file existence and content.
-func (obj *FileRes) contentCheckApply(apply bool) (bool, error) {
-	obj.init.Logf("contentCheckApply(%t)", apply)
-
-	if obj.State == "absent" {
-		if _, err := os.Stat(obj.getPath()); os.IsNotExist(err) {
-			// no such file or directory, but
-			// file should be missing, phew :)
-			return true, nil
-
-		} else if err != nil { // what could this error be?
-			return false, err
-		}
-
-		// state is not okay, no work done, exit, but without error
-		if !apply {
-			return false, nil
-		}
-
-		// apply portion
-		if obj.getPath() == "" || obj.getPath() == "/" {
+		if p == "/" {
 			return false, fmt.Errorf("don't want to remove root") // safety
 		}
-		obj.init.Logf("contentCheckApply: removing: %s", obj.getPath())
+		obj.init.Logf("stateCheckApply: removing: %s", p)
 		// FIXME: respect obj.Recurse here...
 		// TODO: add recurse limit here
-		err := os.RemoveAll(obj.getPath()) // dangerous ;)
-		return false, err                  // either nil or not
+		err := os.RemoveAll(p) // dangerous ;)
+		return false, err      // either nil or not
 	}
 
-	if obj.isDir() && obj.Source == "" {
+	// we need to make a file or a directory now
+
+	if obj.isDir() {
 		return obj.dirCheckApply(apply)
 	}
 
+	// Optimization: we shouldn't even look at obj.Content here, but we can
+	// skip this empty file creation here since we know we're going to be
+	// making it there anyways. This way we save the extra fopen noise.
+	if obj.Content != nil {
+		return false, nil // pretend we actually made it
+	}
+
+	// Create an empty file to ensure one exists. Don't O_TRUNC it, in case
+	// one is magically created right after our exists test. The chmod used
+	// is what is used by the os.Create function.
+	// TODO: is using O_EXCL okay?
+	f, err := os.OpenFile(obj.getPath(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "problem creating empty file")
+	}
+	if err := f.Close(); err != nil {
+		return false, errwrap.Wrapf(err, "problem closing empty file")
+	}
+
+	return false, nil // defer the Content != nil work to later...
+}
+
+// contentCheckApply performs a CheckApply for the file content.
+func (obj *FileRes) contentCheckApply(apply bool) (bool, error) {
+	obj.init.Logf("contentCheckApply(%t)", apply)
+
 	// content is not defined, leave it alone...
-	if obj.Content == nil && obj.Source == "" {
+	if obj.Content == nil {
 		return true, nil
 	}
 
-	if obj.Source == "" { // do the obj.Content checks first...
-		bufferSrc := bytes.NewReader([]byte(*obj.Content))
-		sha256sum, checkOK, err := obj.fileCheckApply(apply, bufferSrc, obj.getPath(), obj.sha256sum)
-		if sha256sum != "" { // empty values mean errored or didn't hash
-			// this can be valid even when the whole function errors
-			obj.sha256sum = sha256sum // cache value
-		}
-		if err != nil {
-			return false, err
-		}
-		// if no err, but !ok, then...
-		return checkOK, nil // success
+	bufferSrc := bytes.NewReader([]byte(*obj.Content))
+	sha256sum, checkOK, err := obj.fileCheckApply(apply, bufferSrc, obj.getPath(), obj.sha256sum)
+	if sha256sum != "" { // empty values mean errored or didn't hash
+		// this can be valid even when the whole function errors
+		obj.sha256sum = sha256sum // cache value
+	}
+	if err != nil {
+		return false, err
+	}
+	// if no err, but !ok, then...
+	return checkOK, nil // success
+}
+
+// sourceCheckApply performs a CheckApply for the file source.
+func (obj *FileRes) sourceCheckApply(apply bool) (bool, error) {
+	obj.init.Logf("sourceCheckApply(%t)", apply)
+
+	// source is not defined, leave it alone...
+	if obj.Source == "" {
+		return true, nil
 	}
 
 	checkOK, err := obj.syncCheckApply(apply, obj.Source, obj.getPath())
 	if err != nil {
-		obj.init.Logf("syncCheckApply: Error: %v", err)
+		obj.init.Logf("syncCheckApply: error: %v", err)
 		return false, err
 	}
 
 	return checkOK, nil
 }
 
-// chmodCheckApply performs a CheckApply for the file permissions.
-func (obj *FileRes) chmodCheckApply(apply bool) (bool, error) {
-	obj.init.Logf("chmodCheckApply(%t)", apply)
-
-	if obj.State == "absent" {
-		// file is absent
-		return true, nil
-	}
-
-	if obj.Mode == "" {
-		// no mode specified, everything is ok
-		return true, nil
-	}
-
-	mode, err := obj.mode()
-
-	// If the file does not exist and we are in
-	// noop mode, do not throw an error.
-	if os.IsNotExist(err) && !apply {
-		return false, nil
-	}
-
-	if err != nil {
-		return false, err
-	}
-
-	fileInfo, err := os.Stat(obj.getPath())
-	if err != nil {
-		return false, err
-	}
-
-	// nothing to do
-	if fileInfo.Mode() == mode {
-		return true, nil
-	}
-
-	// not clean but don't apply
-	if !apply {
-		return false, nil
-	}
-
-	err = os.Chmod(obj.getPath(), mode)
-	return false, err
-}
-
 // chownCheckApply performs a CheckApply for the file ownership.
 func (obj *FileRes) chownCheckApply(apply bool) (bool, error) {
-	var expectedUID, expectedGID int
 	obj.init.Logf("chownCheckApply(%t)", apply)
 
-	if obj.State == "absent" {
-		// file is absent or no owner specified
+	if obj.Owner == "" && obj.Group == "" {
+		// no owner or group specified, everything is ok
 		return true, nil
 	}
 
 	fileInfo, err := os.Stat(obj.getPath())
-
-	// If the file does not exist and we are in
-	// noop mode, do not throw an error.
-	if os.IsNotExist(err) && !apply {
-		return false, nil
-	}
-
-	if err != nil {
+	// TODO: is this a sane behaviour that we want to preserve?
+	// If the file does not exist and we are in noop mode, do not throw an
+	// error.
+	//if os.IsNotExist(err) && !apply {
+	//	return false, nil
+	//}
+	if err != nil { // if the file does not exist, it's correct to error!
 		return false, err
 	}
 
@@ -770,6 +775,8 @@ func (obj *FileRes) chownCheckApply(apply bool) (bool, error) {
 		return false, fmt.Errorf("can't set Owner or Group on this platform")
 	}
 
+	var expectedUID, expectedGID int
+
 	if obj.Owner != "" {
 		expectedUID, err = engineUtil.GetUID(obj.Owner)
 		if err != nil {
@@ -779,7 +786,6 @@ func (obj *FileRes) chownCheckApply(apply bool) (bool, error) {
 		// nothing specified, no changes to be made, expect same as actual
 		expectedUID = int(stUnix.Uid)
 	}
-
 	if obj.Group != "" {
 		expectedGID, err = engineUtil.GetGID(obj.Group)
 		if err != nil {
@@ -803,6 +809,38 @@ func (obj *FileRes) chownCheckApply(apply bool) (bool, error) {
 	return false, os.Chown(obj.getPath(), expectedUID, expectedGID)
 }
 
+// chmodCheckApply performs a CheckApply for the file permissions.
+func (obj *FileRes) chmodCheckApply(apply bool) (bool, error) {
+	obj.init.Logf("chmodCheckApply(%t)", apply)
+
+	if obj.Mode == "" {
+		// no mode specified, everything is ok
+		return true, nil
+	}
+
+	mode, err := obj.mode() // get the desired mode
+	if err != nil {
+		return false, err
+	}
+
+	fileInfo, err := os.Stat(obj.getPath())
+	if err != nil { // if the file does not exist, it's correct to error!
+		return false, err
+	}
+
+	// nothing to do
+	if fileInfo.Mode() == mode {
+		return true, nil
+	}
+
+	// not clean but don't apply
+	if !apply {
+		return false, nil
+	}
+
+	return false, os.Chmod(obj.getPath(), mode)
+}
+
 // CheckApply checks the resource state and applies the resource if the bool
 // input is true. It returns error info and if the state check passed or not.
 func (obj *FileRes) CheckApply(apply bool) (bool, error) {
@@ -820,7 +858,7 @@ func (obj *FileRes) CheckApply(apply bool) (bool, error) {
 
 	checkOK := true
 
-	// always run stateCheckApply before contentCheckApply, they go together
+	// run stateCheckApply before contentCheckApply and sourceCheckApply
 	if c, err := obj.stateCheckApply(apply); err != nil {
 		return false, err
 	} else if !c {
@@ -831,8 +869,7 @@ func (obj *FileRes) CheckApply(apply bool) (bool, error) {
 	} else if !c {
 		checkOK = false
 	}
-
-	if c, err := obj.chmodCheckApply(apply); err != nil {
+	if c, err := obj.sourceCheckApply(apply); err != nil {
 		return false, err
 	} else if !c {
 		checkOK = false
@@ -843,6 +880,11 @@ func (obj *FileRes) CheckApply(apply bool) (bool, error) {
 	} else if !c {
 		checkOK = false
 	}
+	if c, err := obj.chmodCheckApply(apply); err != nil {
+		return false, err
+	} else if !c {
+		checkOK = false
+	}
 
 	return checkOK, nil // w00t
 }
@@ -860,6 +902,11 @@ func (obj *FileRes) Cmp(r engine.Res) error {
 	if obj.getPath() != res.getPath() {
 		return fmt.Errorf("the Path differs")
 	}
+
+	if obj.State != res.State {
+		return fmt.Errorf("the State differs")
+	}
+
 	if (obj.Content == nil) != (res.Content == nil) { // xor
 		return fmt.Errorf("the Content differs")
 	}
@@ -871,9 +918,6 @@ func (obj *FileRes) Cmp(r engine.Res) error {
 	if obj.Source != res.Source {
 		return fmt.Errorf("the Source differs")
 	}
-	if obj.State != res.State {
-		return fmt.Errorf("the State differs")
-	}
 
 	if obj.Owner != res.Owner {
 		return fmt.Errorf("the Owner differs")
@@ -1023,6 +1067,130 @@ func (obj *FileRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
+// Copy copies the resource. Don't call it directly, use engine.ResCopy instead.
+// TODO: should this copy internal state?
+func (obj *FileRes) Copy() engine.CopyableRes {
+	var content *string
+	if obj.Content != nil { // copy the string contents, not the pointer...
+		s := *obj.Content
+		content = &s
+	}
+	return &FileRes{
+		Path:     obj.Path,
+		Dirname:  obj.Dirname,
+		Basename: obj.Basename,
+		State:    obj.State, // TODO: if this becomes a pointer, copy the string!
+		Content:  content,
+		Source:   obj.Source,
+		Owner:    obj.Owner,
+		Group:    obj.Group,
+		Mode:     obj.Mode,
+		Recurse:  obj.Recurse,
+		Force:    obj.Force,
+	}
+}
+
+// Reversed returns the "reverse" or "reciprocal" resource. This is used to
+// "clean" up after a previously defined resource has been removed.
+func (obj *FileRes) Reversed() (engine.ReversibleRes, error) {
+	// NOTE: Previously, we did some more complicated management of reversed
+	// properties. For example, we could add mode and state even when they
+	// weren't originally specified. This code has now been simplified to
+	// avoid this complexity, because it's not really necessary, and it is
+	// somewhat illogical anyways.
+
+	// TODO: reversing this could be tricky, since we'd store it all
+	if obj.isDir() { // XXX: limit this error to a defined state or content?
+		return nil, fmt.Errorf("can't reverse a dir yet")
+	}
+
+	cp, err := engine.ResCopy(obj)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "could not copy")
+	}
+	rev, ok := cp.(engine.ReversibleRes)
+	if !ok {
+		return nil, fmt.Errorf("not reversible")
+	}
+	rev.ReversibleMeta().Disabled = true // the reverse shouldn't run again
+
+	res, ok := cp.(*FileRes)
+	if !ok {
+		return nil, fmt.Errorf("copied res was not our kind")
+	}
+
+	// these are already copied in, and we don't need to change them...
+	//res.Path = obj.Path
+	//res.Dirname = obj.Dirname
+	//res.Basename = obj.Basename
+
+	if obj.State == FileStateExists {
+		res.State = FileStateAbsent
+	}
+	if obj.State == FileStateAbsent {
+		res.State = FileStateExists
+	}
+
+	// If we've specified content, we might need to restore the original, OR
+	// if we're removing the file with a `state => "absent"`, save it too...
+	// The `res.State != FileStateAbsent` check is an optional optimization.
+	if (obj.Content != nil || obj.State == FileStateAbsent) && res.State != FileStateAbsent {
+		content, err := ioutil.ReadFile(obj.getPath())
+		if err != nil && !os.IsNotExist(err) {
+			return nil, errwrap.Wrapf(err, "could not read file for reversal storage")
+		}
+		res.Content = nil
+		if err == nil {
+			str := string(content)
+			res.Content = &str // set contents
+		}
+	}
+	if res.State == FileStateAbsent { // can't specify content when absent!
+		res.Content = nil
+	}
+
+	//res.Source = "" // XXX: what should we do with this?
+	if obj.Source != "" {
+		return nil, fmt.Errorf("can't reverse with Source yet")
+	}
+
+	// There is a race if the operating system is adding/changing/removing
+	// the file between the ioutil.Readfile at the top and here. If there is
+	// a discrepancy between the two, then you might get an unexpected
+	// reverse, but in reality, your perspective is pretty absurd. This is a
+	// user error, and not an issue we actually care about, afaict.
+	fileInfo, err := os.Stat(obj.getPath())
+	if err != nil && !os.IsNotExist(err) {
+		return nil, errwrap.Wrapf(err, "could not stat file for reversal information")
+	}
+	res.Owner = ""
+	res.Group = ""
+	res.Mode = ""
+	if err == nil {
+		stUnix, ok := fileInfo.Sys().(*syscall.Stat_t)
+		// XXX: add a !ok error scenario or some alternative?
+		if ok { // if not, this isn't unix
+			if obj.Owner != "" {
+				res.Owner = strconv.FormatInt(int64(stUnix.Uid), 10) // Uid is a uint32
+			}
+			if obj.Group != "" {
+				res.Group = strconv.FormatInt(int64(stUnix.Gid), 10) // Gid is a uint32
+			}
+		}
+
+		// TODO: use Mode().String() when we support full rwx style mode specs!
+		if obj.Mode != "" {
+			res.Mode = fmt.Sprintf("%#o", fileInfo.Mode().Perm()) // 0400, 0777, etc.
+		}
+	}
+
+	// these are already copied in, and we don't need to change them...
+	//res.Recurse = obj.Recurse
+	//res.Force = obj.Force
+
+	return res, nil
+}
+
 // smartPath adds a trailing slash to the path if it is a directory.
 func smartPath(fileInfo os.FileInfo) string {
 	smartPath := fileInfo.Name() // absolute path

engine/resources/file_test.go

@@ -78,7 +78,7 @@ func TestMiscEncodeDecode1(t *testing.T) {
 	e := gob.NewEncoder(&b1)
 	err = e.Encode(&input) // pass with &
 	if err != nil {
-		t.Errorf("Gob failed to Encode: %v", err)
+		t.Errorf("gob failed to Encode: %v", err)
 	}
 	str := base64.StdEncoding.EncodeToString(b1.Bytes())
 
@@ -86,27 +86,27 @@ func TestMiscEncodeDecode1(t *testing.T) {
 	var output interface{}
 	bb, err := base64.StdEncoding.DecodeString(str)
 	if err != nil {
-		t.Errorf("Base64 failed to Decode: %v", err)
+		t.Errorf("base64 failed to Decode: %v", err)
 	}
 	b2 := bytes.NewBuffer(bb)
 	d := gob.NewDecoder(b2)
 	err = d.Decode(&output) // pass with &
 	if err != nil {
-		t.Errorf("Gob failed to Decode: %v", err)
+		t.Errorf("gob failed to Decode: %v", err)
 	}
 
 	res1, ok := input.(engine.Res)
 	if !ok {
-		t.Errorf("Input %v is not a Res", res1)
+		t.Errorf("input %v is not a Res", res1)
 		return
 	}
 	res2, ok := output.(engine.Res)
 	if !ok {
-		t.Errorf("Output %v is not a Res", res2)
+		t.Errorf("output %v is not a Res", res2)
 		return
 	}
 	if err := res1.Cmp(res2); err != nil {
-		t.Errorf("The input and output Res values do not match: %+v", err)
+		t.Errorf("the input and output Res values do not match: %+v", err)
 	}
 }
 
@@ -116,7 +116,7 @@ func TestMiscEncodeDecode2(t *testing.T) {
 	// encode
 	input, err := engine.NewNamedResource("file", "file1")
 	if err != nil {
-		t.Errorf("Can't create: %v", err)
+		t.Errorf("can't create: %v", err)
 		return
 	}
 	// NOTE: Do not add this bit of code, because it would cause the path to
@@ -128,29 +128,29 @@ func TestMiscEncodeDecode2(t *testing.T) {
 
 	b64, err := engineUtil.ResToB64(input)
 	if err != nil {
-		t.Errorf("Can't encode: %v", err)
+		t.Errorf("can't encode: %v", err)
 		return
 	}
 
 	output, err := engineUtil.B64ToRes(b64)
 	if err != nil {
-		t.Errorf("Can't decode: %v", err)
+		t.Errorf("can't decode: %v", err)
 		return
 	}
 
 	res1, ok := input.(engine.Res)
 	if !ok {
-		t.Errorf("Input %v is not a Res", res1)
+		t.Errorf("input %v is not a Res", res1)
 		return
 	}
 	res2, ok := output.(engine.Res)
 	if !ok {
-		t.Errorf("Output %v is not a Res", res2)
+		t.Errorf("output %v is not a Res", res2)
 		return
 	}
 	// this uses the standalone file cmp function
 	if err := res1.Cmp(res2); err != nil {
-		t.Errorf("The input and output Res values do not match: %+v", err)
+		t.Errorf("the input and output Res values do not match: %+v", err)
 	}
 }
 
@@ -160,7 +160,7 @@ func TestMiscEncodeDecode3(t *testing.T) {
 	// encode
 	input, err := engine.NewNamedResource("file", "file1")
 	if err != nil {
-		t.Errorf("Can't create: %v", err)
+		t.Errorf("can't create: %v", err)
 		return
 	}
 	fileRes := input.(*FileRes) // must not panic
@@ -169,29 +169,82 @@ func TestMiscEncodeDecode3(t *testing.T) {
 
 	b64, err := engineUtil.ResToB64(input)
 	if err != nil {
-		t.Errorf("Can't encode: %v", err)
+		t.Errorf("can't encode: %v", err)
 		return
 	}
 
 	output, err := engineUtil.B64ToRes(b64)
 	if err != nil {
-		t.Errorf("Can't decode: %v", err)
+		t.Errorf("can't decode: %v", err)
 		return
 	}
 
 	res1, ok := input.(engine.Res)
 	if !ok {
-		t.Errorf("Input %v is not a Res", res1)
+		t.Errorf("input %v is not a Res", res1)
 		return
 	}
 	res2, ok := output.(engine.Res)
 	if !ok {
-		t.Errorf("Output %v is not a Res", res2)
+		t.Errorf("output %v is not a Res", res2)
 		return
 	}
 	// this uses the more complete, engine cmp function
 	if err := engine.ResCmp(res1, res2); err != nil {
-		t.Errorf("The input and output Res values do not match: %+v", err)
+		t.Errorf("the input and output Res values do not match: %+v", err)
+	}
+}
+
+func TestMiscEncodeDecode4(t *testing.T) {
+	var err error
+	const (
+		Kind = "file"
+		Name = "file1"
+	)
+
+	// encode
+	input, err := engine.NewNamedResource(Kind, Name)
+	if err != nil {
+		t.Errorf("can't create: %v", err)
+		return
+	}
+	fileRes := input.(*FileRes) // must not panic
+	fileRes.Path = "/tmp/whatever"
+	// TODO: add other params/traits/etc here!
+
+	b64, err := engineUtil.ResToB64(input)
+	if err != nil {
+		t.Errorf("can't encode: %v", err)
+		return
+	}
+
+	output, err := engineUtil.B64ToRes(b64)
+	if err != nil {
+		t.Errorf("can't decode: %v", err)
+		return
+	}
+
+	res1, ok := input.(engine.Res)
+	if !ok {
+		t.Errorf("input %v is not a Res", res1)
+		return
+	}
+	res2, ok := output.(engine.Res)
+	if !ok {
+		t.Errorf("output %v is not a Res", res2)
+		return
+	}
+	// this uses the more complete, engine cmp function
+	if err := engine.ResCmp(res1, res2); err != nil {
+		t.Errorf("the input and output Res values do not match: %+v", err)
+	}
+
+	// ensure the kind and name are correctly decoded too!
+	if kind := res2.Kind(); kind != Kind {
+		t.Errorf("the output kind was `%s`, expected `%s`", kind, Kind)
+	}
+	if name := res2.Name(); name != Name {
+		t.Errorf("the output name was `%s`, expected `%s`", name, Name)
 	}
 }
 

engine/resources/group.go

@@ -58,7 +58,7 @@ func (obj *GroupRes) Default() engine.Res {
 // Validate if the params passed in are valid data.
 func (obj *GroupRes) Validate() error {
 	if obj.State != "exists" && obj.State != "absent" {
-		return fmt.Errorf("State must be 'exists' or 'absent'")
+		return fmt.Errorf("state must be 'exists' or 'absent'")
 	}
 	return nil
 }
@@ -220,32 +220,24 @@ func (obj *GroupRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *GroupRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *GroupRes) Compare(r engine.Res) bool {
 	// we can only compare GroupRes to others of the same resource kind
 	res, ok := r.(*GroupRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.State != res.State {
-		return false
+		return fmt.Errorf("the State differs")
 	}
 	if (obj.GID == nil) != (res.GID == nil) {
-		return false
+		return fmt.Errorf("the GID differs")
 	}
 	if obj.GID != nil && res.GID != nil {
 		if *obj.GID != *res.GID {
-			return false
+			return fmt.Errorf("the GID differs")
 		}
 	}
-	return true
+	return nil
 }
 
 // GroupUID is the UID struct for GroupRes.

engine/resources/hostname.go

@@ -219,31 +219,23 @@ func (obj *HostnameRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *HostnameRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *HostnameRes) Compare(r engine.Res) bool {
 	// we can only compare HostnameRes to others of the same resource kind
 	res, ok := r.(*HostnameRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.PrettyHostname != res.PrettyHostname {
-		return false
+		return fmt.Errorf("the PrettyHostname differs")
 	}
 	if obj.StaticHostname != res.StaticHostname {
-		return false
+		return fmt.Errorf("the StaticHostname differs")
 	}
 	if obj.TransientHostname != res.TransientHostname {
-		return false
+		return fmt.Errorf("the TransientHostname differs")
 	}
 
-	return true
+	return nil
 }
 
 // HostnameUID is the UID struct for HostnameRes.

engine/resources/mount.go

@@ -75,6 +75,8 @@ const (
 	// diskByLabel is the location of symlinks for partitions by label.
 	diskByPartLabel = devDisk + "by-partlabel/"
 
+	// dbusSystemdService is the service to connect to systemd itself.
+	dbusSystemd1Service = "org.freedesktop.systemd1"
 	// dbusSystemd1Interface is the base systemd1 path.
 	dbusSystemd1Path = "/org/freedesktop/systemd1"
 	// dbusUnitPath is the dbus path where mount unit files are found.
@@ -88,6 +90,9 @@ const (
 	dbusManagerInterface = dbusSystemd1Interface + ".Manager"
 	// dbusRestartUnit is the dbus method for restarting systemd units.
 	dbusRestartUnit = dbusManagerInterface + ".RestartUnit"
+	// dbusReloadSystemd is the dbus method for reloading systemd settings.
+	// (i.e. systemctl daemon-reload)
+	dbusReloadSystemd = dbusManagerInterface + ".Reload"
 	// restartTimeout is the delay before restartUnit is assumed to have
 	// failed.
 	dbusRestartCtxTimeout = 10
@@ -576,7 +581,10 @@ func mountReload() error {
 	}
 	defer conn.Close()
 	// systemctl daemon-reload
-	conn.BusObject().Call("Reload", 0)
+	call := conn.Object(dbusSystemd1Service, dbusSystemd1Path).Call(dbusReloadSystemd, 0)
+	if call.Err != nil {
+		return errwrap.Wrapf(call.Err, "error reloading systemd")
+	}
 
 	// systemctl restart local-fs.target
 	if err := restartUnit(conn, "local-fs.target"); err != nil {
@@ -619,7 +627,7 @@ func restartUnit(conn *dbus.Conn, unit string) error {
 	defer conn.RemoveSignal(ch)
 
 	// restart the unit
-	sd1 := conn.Object(dbusSystemd1Interface, dbus.ObjectPath(dbusSystemd1Path))
+	sd1 := conn.Object(dbusSystemd1Service, dbus.ObjectPath(dbusSystemd1Path))
 	if call := sd1.Call(dbusRestartUnit, 0, unit, "fail"); call.Err != nil {
 		return errwrap.Wrapf(call.Err, "error restarting unit: %s", unit)
 	}

engine/resources/msg.go

@@ -200,36 +200,28 @@ func (obj *MsgRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *MsgRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *MsgRes) Compare(r engine.Res) bool {
 	// we can only compare MsgRes to others of the same resource kind
 	res, ok := r.(*MsgRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.Body != res.Body {
-		return false
+		return fmt.Errorf("the Body differs")
 	}
 	if obj.Priority != res.Priority {
-		return false
+		return fmt.Errorf("the Priority differs")
 	}
 	if len(obj.Fields) != len(res.Fields) {
-		return false
+		return fmt.Errorf("the length of Fields differs")
 	}
 	for field, value := range obj.Fields {
 		if res.Fields[field] != value {
-			return false
+			return fmt.Errorf("the Fields differ")
 		}
 	}
 
-	return true
+	return nil
 }
 
 // MsgUID is a unique representation for a MsgRes object.

engine/resources/net.go

@@ -506,34 +506,26 @@ func (obj *NetRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *NetRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *NetRes) Compare(r engine.Res) bool {
 	// we can only compare NetRes to others of the same resource kind
 	res, ok := r.(*NetRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.State != res.State {
-		return false
+		return fmt.Errorf("the State differs")
 	}
 	if (obj.Addrs == nil) != (res.Addrs == nil) {
-		return false
+		return fmt.Errorf("the Addrs differ")
 	}
 	if err := util.SortedStrSliceCompare(obj.Addrs, res.Addrs); err != nil {
-		return false
+		return fmt.Errorf("the Addrs differ")
 	}
 	if obj.Gateway != res.Gateway {
-		return false
+		return fmt.Errorf("the Gateway differs")
 	}
 
-	return true
+	return nil
 }
 
 // NetUID is a unique resource identifier.

engine/resources/nspawn.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"strconv"
+	"strings"
 	"unicode"
 
 	"github.com/purpleidea/mgmt/engine"
@@ -260,35 +261,27 @@ func (obj *NspawnRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *NspawnRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *NspawnRes) Compare(r engine.Res) bool {
 	// we can only compare NspawnRes to others of the same resource kind
 	res, ok := r.(*NspawnRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.State != res.State {
-		return false
+		return fmt.Errorf("the State differs")
 	}
 
 	// TODO: why is res.svc ever nil?
 	if (obj.svc == nil) != (res.svc == nil) { // xor
-		return false
+		return fmt.Errorf("the svc differs")
 	}
 	if obj.svc != nil && res.svc != nil {
-		if !obj.svc.Compare(res.svc) {
-			return false
+		if err := obj.svc.Cmp(res.svc); err != nil {
+			return errwrap.Wrapf(err, "the svc differs")
 		}
 	}
 
-	return true
+	return nil
 }
 
 // NspawnUID is a unique resource identifier.
@@ -358,10 +351,12 @@ func systemdVersion() (uint16, error) {
 		return 0, errwrap.Wrapf(err, "could not get version property")
 	}
 	// lose the surrounding quotes
-	verNum, err := strconv.Unquote(verString)
+	verNumString, err := strconv.Unquote(verString)
 	if err != nil {
 		return 0, errwrap.Wrapf(err, "error unquoting version number")
 	}
+	// trim possible version suffix like in "242.19-1"
+	verNum := strings.Split(verNumString, ".")[0]
 	// cast to uint16
 	ver, err := strconv.ParseUint(verNum, 10, 16)
 	if err != nil {

engine/resources/packagekit/packagekit.go

@@ -352,7 +352,7 @@ loop:
 				// should already be broken
 				break loop
 			} else {
-				return []string{}, fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return []string{}, fmt.Errorf("error in body: %v", signal.Body)
 			}
 		}
 	}
@@ -363,9 +363,9 @@ loop:
 func (obj *Conn) IsInstalledList(packages []string) ([]bool, error) {
 	var filter uint64          // initializes at the "zero" value of 0
 	filter += PkFilterEnumArch // always search in our arch
-	packageIDs, e := obj.ResolvePackages(packages, filter)
-	if e != nil {
-		return nil, fmt.Errorf("ResolvePackages error: %v", e)
+	packageIDs, err := obj.ResolvePackages(packages, filter)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "error resolving packages")
 	}
 
 	var m = make(map[string]int)
@@ -443,7 +443,7 @@ loop:
 			}
 
 			if signal.Name == FmtTransactionMethod("ErrorCode") {
-				return fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return fmt.Errorf("error in body: %v", signal.Body)
 			} else if signal.Name == FmtTransactionMethod("Package") {
 				// a package was installed...
 				// only start the timer once we're here...
@@ -454,14 +454,14 @@ loop:
 			} else if signal.Name == FmtTransactionMethod("Destroy") {
 				return nil // success
 			} else {
-				return fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return fmt.Errorf("error in body: %v", signal.Body)
 			}
 		case <-util.TimeAfterOrBlock(timeout):
 			if finished {
 				obj.Logf("Timeout: InstallPackages: Waiting for 'Destroy'")
 				return nil // got tired of waiting for Destroy
 			}
-			return fmt.Errorf("PackageKit: Timeout: InstallPackages: %s", strings.Join(packageIDs, ", "))
+			return fmt.Errorf("timeout installing packages: %s", strings.Join(packageIDs, ", "))
 		}
 	}
 }
@@ -500,7 +500,7 @@ loop:
 			}
 
 			if signal.Name == FmtTransactionMethod("ErrorCode") {
-				return fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return fmt.Errorf("error in body: %v", signal.Body)
 			} else if signal.Name == FmtTransactionMethod("Package") {
 				// a package was installed...
 				continue loop
@@ -511,7 +511,7 @@ loop:
 				// should already be broken
 				break loop
 			} else {
-				return fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return fmt.Errorf("error in body: %v", signal.Body)
 			}
 		}
 	}
@@ -549,7 +549,7 @@ loop:
 			}
 
 			if signal.Name == FmtTransactionMethod("ErrorCode") {
-				return fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return fmt.Errorf("error in body: %v", signal.Body)
 			} else if signal.Name == FmtTransactionMethod("Package") {
 			} else if signal.Name == FmtTransactionMethod("Finished") {
 				// TODO: should we wait for the Destroy signal?
@@ -558,7 +558,7 @@ loop:
 				// should already be broken
 				break loop
 			} else {
-				return fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return fmt.Errorf("error in body: %v", signal.Body)
 			}
 		}
 	}
@@ -601,7 +601,7 @@ loop:
 			}
 
 			if signal.Name == FmtTransactionMethod("ErrorCode") {
-				err = fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				err = fmt.Errorf("error in body: %v", signal.Body)
 				return
 
 				// one signal returned per packageID found...
@@ -626,7 +626,7 @@ loop:
 				// should already be broken
 				break loop
 			} else {
-				err = fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				err = fmt.Errorf("error in body: %v", signal.Body)
 				return
 			}
 		}
@@ -669,7 +669,7 @@ loop:
 			}
 
 			if signal.Name == FmtTransactionMethod("ErrorCode") {
-				return nil, fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return nil, fmt.Errorf("error in body: %v", signal.Body)
 			} else if signal.Name == FmtTransactionMethod("Package") {
 
 				//pkg_int, ok := signal.Body[0].(int)
@@ -692,7 +692,7 @@ loop:
 				// should already be broken
 				break loop
 			} else {
-				return nil, fmt.Errorf("PackageKit: Error: %v", signal.Body)
+				return nil, fmt.Errorf("error in body: %v", signal.Body)
 			}
 		}
 	}
@@ -718,9 +718,9 @@ func (obj *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 	if obj.Debug {
 		obj.Logf("PackagesToPackageIDs(): %s", strings.Join(packages, ", "))
 	}
-	resolved, e := obj.ResolvePackages(packages, filter)
-	if e != nil {
-		return nil, fmt.Errorf("Resolve error: %v", e)
+	resolved, err := obj.ResolvePackages(packages, filter)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "error resolving")
 	}
 
 	found := make([]bool, count) // default false
@@ -758,7 +758,7 @@ func (obj *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 		}
 		state := packageMap[pkg] // lookup the requested state/version
 		if state == "" {
-			return nil, fmt.Errorf("Empty package state for %v", pkg)
+			return nil, fmt.Errorf("empty package state for: `%s`", pkg)
 		}
 		found[index] = true
 		stateIsVersion := (state != "installed" && state != "uninstalled" && state != "newest") // must be a ver. string
@@ -794,9 +794,9 @@ func (obj *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 	// to be done, and if so, anything that needs updating isn't newest!
 	// if something isn't installed, we can't verify it with this method
 	// FIXME: https://github.com/hughsie/PackageKit/issues/116
-	updates, e := obj.GetUpdates(filter)
-	if e != nil {
-		return nil, fmt.Errorf("Updates error: %v", e)
+	updates, err := obj.GetUpdates(filter)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "updates error")
 	}
 	for _, packageID := range updates {
 		//obj.Logf("* %v", packageID)
@@ -844,9 +844,9 @@ func (obj *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 			if obj.Debug {
 				obj.Logf("PackagesToPackageIDs(): Recurse: %s", strings.Join(checkPackages, ", "))
 			}
-			recursion, e = obj.PackagesToPackageIDs(filteredPackageMap, filter+PkFilterEnumNewest)
-			if e != nil {
-				return nil, fmt.Errorf("Recursion error: %v", e)
+			recursion, err = obj.PackagesToPackageIDs(filteredPackageMap, filter+PkFilterEnumNewest)
+			if err != nil {
+				return nil, errwrap.Wrapf(err, "recursion error")
 			}
 		}
 	}

engine/resources/password.go

@@ -295,33 +295,25 @@ func (obj *PasswordRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *PasswordRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *PasswordRes) Compare(r engine.Res) bool {
 	// we can only compare PasswordRes to others of the same resource kind
 	res, ok := r.(*PasswordRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.Length != res.Length {
-		return false
+		return fmt.Errorf("the Length differs")
 	}
 	// TODO: we *could* optimize by allowing CheckApply to move from
 	// saved->!saved, by removing the file, but not likely worth it!
 	if obj.Saved != res.Saved {
-		return false
+		return fmt.Errorf("the Saved differs")
 	}
 	if obj.CheckRecovery != res.CheckRecovery {
-		return false
+		return fmt.Errorf("the CheckRecovery differs")
 	}
 
-	return true
+	return nil
 }
 
 // PasswordUID is the UID struct for PasswordRes.

engine/resources/print.go

@@ -115,24 +115,16 @@ func (obj *PrintRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *PrintRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *PrintRes) Compare(r engine.Res) bool {
 	// we can only compare PrintRes to others of the same resource kind
 	res, ok := r.(*PrintRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.Msg != res.Msg {
-		return false
+		return fmt.Errorf("the Msg differs")
 	}
-	return true
+	return nil
 }
 
 // PrintUID is the UID struct for PrintRes.

engine/resources/resources_test.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/util"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // TODO: consider providing this as a lib so that we can add tests into the
@@ -152,6 +153,45 @@ func NewClearChangedStep(ms uint) Step {
 	}
 }
 
+// FileExpect takes a path and a string to expect in that file, and builds a
+// Step that checks that out of them.
+func FileExpect(p, s string) Step { // path & string
+	return &manualStep{
+		action: func() error { return nil },
+		expect: func() error {
+			content, err := ioutil.ReadFile(p)
+			if err != nil {
+				return err
+			}
+			if string(content) != s {
+				return fmt.Errorf("contents did not match in %s", p)
+			}
+			return nil
+		},
+	}
+}
+
+// FileExpect takes a path and a string to write to that file, and builds a Step
+// that does that to them.
+func FileWrite(p, s string) Step { // path & string
+	return &manualStep{
+		action: func() error {
+			// TODO: apparently using 0666 is equivalent to respecting the current umask
+			const umask = 0666
+			return ioutil.WriteFile(p, []byte(s), umask)
+		},
+		expect: func() error { return nil },
+	}
+}
+
+// ErrIsNotExistOK returns nil if we get an IsNotExist true result on the error.
+func ErrIsNotExistOK(e error) error {
+	if os.IsNotExist(e) {
+		return nil
+	}
+	return errwrap.Wrapf(e, "unexpected error")
+}
+
 func TestResources1(t *testing.T) {
 	type test struct { // an individual test
 		name      string
@@ -177,31 +217,6 @@ func TestResources1(t *testing.T) {
 			expect: func() error { return nil },
 		}
 	}
-	fileExpect := func(p, s string) Step { // path & string
-		return &manualStep{
-			action: func() error { return nil },
-			expect: func() error {
-				content, err := ioutil.ReadFile(p)
-				if err != nil {
-					return err
-				}
-				if string(content) != s {
-					return fmt.Errorf("contents did not match in %s", p)
-				}
-				return nil
-			},
-		}
-	}
-	fileWrite := func(p, s string) Step { // path & string
-		return &manualStep{
-			action: func() error {
-				// TODO: apparently using 0666 is equivalent to respecting the current umask
-				const umask = 0666
-				return ioutil.WriteFile(p, []byte(s), umask)
-			},
-			expect: func() error { return nil },
-		}
-	}
 
 	testCases := []test{}
 	{
@@ -210,17 +225,18 @@ func TestResources1(t *testing.T) {
 		p := "/tmp/whatever"
 		s := "hello, world\n"
 		res.Path = p
+		res.State = "exists"
 		contents := s
 		res.Content = &contents
 
 		timeline := []Step{
 			NewStartupStep(1000 * 60),          // startup
 			NewChangedStep(1000*60, false),     // did we do something?
-			fileExpect(p, s),                   // check initial state
+			FileExpect(p, s),                   // check initial state
 			NewClearChangedStep(1000 * 15),     // did we do something?
-			fileWrite(p, "this is whatever\n"), // change state
+			FileWrite(p, "this is whatever\n"), // change state
 			NewChangedStep(1000*60, false),     // did we do something?
-			fileExpect(p, s),                   // check again
+			FileExpect(p, s),                   // check again
 			sleep(1),                           // we can sleep too!
 		}
 
@@ -249,11 +265,11 @@ func TestResources1(t *testing.T) {
 		timeline := []Step{
 			NewStartupStep(1000 * 60),        // startup
 			NewChangedStep(1000*60, false),   // did we do something?
-			fileExpect(f, s+"\n"),            // check initial state
+			FileExpect(f, s+"\n"),            // check initial state
 			NewClearChangedStep(1000 * 15),   // did we do something?
-			fileWrite(f, "this is stuff!\n"), // change state
+			FileWrite(f, "this is stuff!\n"), // change state
 			NewChangedStep(1000*60, false),   // did we do something?
-			fileExpect(f, s+"\n"),            // check again
+			FileExpect(f, s+"\n"),            // check again
 			sleep(1),                         // we can sleep too!
 		}
 
@@ -278,7 +294,7 @@ func TestResources1(t *testing.T) {
 		timeline := []Step{
 			NewStartupStep(1000 * 60),      // startup
 			NewChangedStep(1000*60, false), // did we do something?
-			fileExpect(p, ""),              // check initial state
+			FileExpect(p, ""),              // check initial state
 			NewClearChangedStep(1000 * 15), // did we do something?
 		}
 
@@ -303,7 +319,7 @@ func TestResources1(t *testing.T) {
 		timeline := []Step{
 			NewStartupStep(1000 * 60),     // startup
 			NewChangedStep(1000*60, true), // did we do something?
-			fileExpect(p, content),        // check initial state
+			FileExpect(p, content),        // check initial state
 		}
 
 		testCases = append(testCases, test{
@@ -372,7 +388,7 @@ func TestResources1(t *testing.T) {
 			doneChan := make(chan struct{})
 			debug := testing.Verbose() // set via the -test.v flag to `go test`
 			logf := func(format string, v ...interface{}) {
-				t.Logf(fmt.Sprintf("test #%d: Res: ", index)+format, v...)
+				t.Logf(fmt.Sprintf("test #%d: ", index)+format, v...)
 			}
 			init := &engine.Init{
 				Running: func() {
@@ -548,3 +564,619 @@ func TestResources1(t *testing.T) {
 		})
 	}
 }
+
+// TestResources2 just tests a partial execution of the resource by running
+// CheckApply and Reverse and basics without the mainloop. It's a less accurate
+// representation of a running resource, but is still useful for many
+// circumstances. This also uses a simpler timeline, because it was not possible
+// to get the reference passing of the reversed resource working with the fancy
+// version.
+func TestResources2(t *testing.T) {
+	type test struct { // an individual test
+		name     string
+		timeline []func() error // TODO: this could be a generator that keeps pushing out steps until it's done!
+		expect   func() error   // function to check for expected state
+		startup  func() error   // function to run as startup (unused?)
+		cleanup  func() error   // function to run as cleanup
+	}
+
+	// resValidate runs Validate on the res.
+	resValidate := func(res engine.Res) func() error {
+		// run Close
+		return func() error {
+			return res.Validate()
+		}
+	}
+	// resInit runs Init on the res.
+	resInit := func(res engine.Res) func() error {
+		logf := func(format string, v ...interface{}) {
+			// noop for now
+		}
+		init := &engine.Init{
+			//Debug: debug,
+			Logf: logf,
+
+			// unused
+			Send: func(st interface{}) error {
+				return nil
+			},
+			Recv: func() map[string]*engine.Send {
+				return map[string]*engine.Send{}
+			},
+		}
+		// run Init
+		return func() error {
+			return res.Init(init)
+
+		}
+	}
+	// resCheckApplyError runs CheckApply with noop = false for the res. It
+	// errors if the returned checkOK values isn't what we were expecting or
+	// if the errOK function returns an error when given a chance to inspect
+	// the returned error.
+	resCheckApplyError := func(res engine.Res, expCheckOK bool, errOK func(e error) error) func() error {
+		return func() error {
+			checkOK, err := res.CheckApply(true) // no noop!
+			if e := errOK(err); e != nil {
+				return errwrap.Wrapf(e, "error from CheckApply did not match expected")
+			}
+			if checkOK != expCheckOK {
+				return fmt.Errorf("result from CheckApply did not match expected: `%t` != `%t`", checkOK, expCheckOK)
+			}
+			return nil
+		}
+	}
+	// resCheckApply runs CheckApply with noop = false for the res. It
+	// errors if the returned checkOK values isn't what we were expecting or
+	// if there was an error.
+	resCheckApply := func(res engine.Res, expCheckOK bool) func() error {
+		errOK := func(e error) error {
+			if e == nil {
+				return nil
+			}
+			return errwrap.Wrapf(e, "unexpected error from CheckApply")
+		}
+		return resCheckApplyError(res, expCheckOK, errOK)
+	}
+	// resClose runs Close on the res.
+	resClose := func(res engine.Res) func() error {
+		// run Close
+		return func() error {
+			return res.Close()
+		}
+	}
+	// resReversal runs Reverse on the resource and stores the result in the
+	// rev variable. This should be called before the res CheckApply, and
+	// usually before Init, but after Validate.
+	resReversal := func(res engine.Res, rev *engine.Res) func() error {
+		return func() error {
+			r, ok := res.(engine.ReversibleRes)
+			if !ok {
+				return fmt.Errorf("res is not a ReversibleRes")
+			}
+
+			// We don't really need this to be checked here.
+			//if r.ReversibleMeta().Disabled {
+			//	return fmt.Errorf("res did not specify Meta:reverse")
+			//}
+
+			if r.ReversibleMeta().Reversal {
+				//logf("triangle reversal") // warn!
+			}
+
+			reversed, err := r.Reversed()
+			if err != nil {
+				return errwrap.Wrapf(err, "could not reverse: %s", r.String())
+			}
+			if reversed == nil {
+				return nil // this can't be reversed, or isn't implemented here
+			}
+
+			reversed.ReversibleMeta().Reversal = true // set this for later...
+
+			retRes, ok := reversed.(engine.Res)
+			if !ok {
+				return fmt.Errorf("not a Res")
+			}
+
+			*rev = retRes // store!
+			return nil
+		}
+	}
+	fileWrite := func(p, s string) func() error {
+		// write the file to path
+		return func() error {
+			return ioutil.WriteFile(p, []byte(s), 0666)
+		}
+	}
+	fileExpect := func(p, s string) func() error {
+		// check the contents at the path match the string we expect
+		return func() error {
+			content, err := ioutil.ReadFile(p)
+			if err != nil {
+				return err
+			}
+			if string(content) != s {
+				return fmt.Errorf("contents did not match in %s", p)
+			}
+			return nil
+		}
+	}
+	fileAbsent := func(p string) func() error {
+		// does the file exist?
+		return func() error {
+			_, err := os.Stat(p)
+			if !os.IsNotExist(err) {
+				return fmt.Errorf("file was supposed to be absent, got: %+v", err)
+			}
+			return nil
+		}
+	}
+	fileRemove := func(p string) func() error {
+		// remove the file at path
+		return func() error {
+			err := os.Remove(p)
+			// if the file isn't there, don't error
+			if err != nil && !os.IsNotExist(err) {
+				return err
+			}
+			return nil
+		}
+	}
+
+	testCases := []test{}
+	{
+		//file "/tmp/somefile" {
+		//	state => "exists",
+		//	content => "some new text\n",
+		//}
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somefile"
+		res.Path = p
+		res.State = "exists"
+		content := "some new text\n"
+		res.Content = &content
+
+		timeline := []func() error{
+			fileWrite(p, "whatever"),
+			resValidate(r1),
+			resInit(r1),
+			resCheckApply(r1, false), // changed
+			fileExpect(p, content),
+			resCheckApply(r1, true), // it's already good
+			resClose(r1),
+			fileExpect(p, content), // ensure it exists
+		}
+
+		testCases = append(testCases, test{
+			name:     "simple file",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return nil },
+		})
+	}
+	{
+		//file "/tmp/somefile" {
+		//	# state is NOT specified
+		//	content => "some new text\n",
+		//}
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somefile"
+		res.Path = p
+		//res.State = "exists" // not specified!
+		content := "some new text\n"
+		res.Content = &content
+
+		timeline := []func() error{
+			fileWrite(p, "whatever"),
+			resValidate(r1),
+			resInit(r1),
+			resCheckApply(r1, false), // changed
+			fileExpect(p, content),
+			resCheckApply(r1, true), // it's already good
+			resClose(r1),
+			fileExpect(p, content), // ensure it exists
+		}
+
+		testCases = append(testCases, test{
+			name:     "edit file only",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return nil },
+		})
+	}
+	{
+		//file "/tmp/somefile" {
+		//	# state is NOT specified
+		//	content => "some new text\n",
+		//}
+		// and no existing file exists! (therefore we want an error!)
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somefile"
+		res.Path = p
+		//res.State = "exists" // not specified!
+		content := "some new text\n"
+		res.Content = &content
+
+		timeline := []func() error{
+			fileRemove(p), // nothing here
+			resValidate(r1),
+			resInit(r1),
+			resCheckApplyError(r1, false, ErrIsNotExistOK), // should error
+			resCheckApplyError(r1, false, ErrIsNotExistOK), // double check
+			resClose(r1),
+			fileAbsent(p), // ensure it's absent
+		}
+
+		testCases = append(testCases, test{
+			name:     "strict file",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return nil },
+		})
+	}
+	{
+		//file "/tmp/somefile" {
+		//	state => "absent",
+		//}
+		// and no existing file exists!
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somefile"
+		res.Path = p
+		res.State = "absent"
+
+		timeline := []func() error{
+			fileRemove(p), // nothing here
+			resValidate(r1),
+			resInit(r1),
+			resCheckApply(r1, true),
+			resCheckApply(r1, true),
+			resClose(r1),
+			fileAbsent(p), // ensure it's absent
+		}
+
+		testCases = append(testCases, test{
+			name:     "absent file",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return nil },
+		})
+	}
+	{
+		//file "/tmp/somefile" {
+		//	state => "absent",
+		//}
+		// and a file already exists!
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somefile"
+		res.Path = p
+		res.State = "absent"
+
+		timeline := []func() error{
+			fileWrite(p, "whatever"),
+			resValidate(r1),
+			resInit(r1),
+			resCheckApply(r1, false),
+			resCheckApply(r1, true),
+			resClose(r1),
+			fileAbsent(p), // ensure it's absent
+		}
+
+		testCases = append(testCases, test{
+			name:     "absent file pre-existing",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return nil },
+		})
+	}
+	{
+		//file "/tmp/somefile" {
+		//	content => "some new text\n",
+		//	state => "exists",
+		//
+		//	Meta:reverse => true,
+		//}
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somefile"
+		res.Path = p
+		res.State = "exists"
+		content := "some new text\n"
+		res.Content = &content
+		original := "this is the original state\n" // original state
+		var r2 engine.Res                          // future reversed resource
+
+		timeline := []func() error{
+			fileWrite(p, original),
+			fileExpect(p, original),
+			resValidate(r1),
+			resReversal(r1, &r2), // runs in Init to snapshot
+			func() error { // random test
+				if st := r2.(*FileRes).State; st != "absent" {
+					return fmt.Errorf("unexpected state: %s", st)
+				}
+				return nil
+			},
+			resInit(r1),
+			resCheckApply(r1, false), // changed
+			fileExpect(p, content),
+			resCheckApply(r1, true), // it's already good
+			resClose(r1),
+			//resValidate(r2), // no!!!
+			func() error {
+				// wrap it b/c it is currently nil
+				return r2.Validate()
+			},
+			func() error {
+				return resInit(r2)()
+			},
+			func() error {
+				return resCheckApply(r2, false)()
+			},
+			func() error {
+				return resCheckApply(r2, true)()
+			},
+			func() error {
+				return resClose(r2)()
+			},
+			fileAbsent(p), // ensure it's absent
+		}
+
+		testCases = append(testCases, test{
+			name:     "some file",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return nil },
+		})
+	}
+	{
+		//file "/tmp/somefile" {
+		//	content => "some new text\n",
+		//
+		//	Meta:reverse => true,
+		//}
+		//# and there's an existing file at this path...
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somefile"
+		res.Path = p
+		//res.State = "exists" // unspecified
+		content := "some new text\n"
+		res.Content = &content
+		original := "this is the original state\n" // original state
+		var r2 engine.Res                          // future reversed resource
+
+		timeline := []func() error{
+			fileWrite(p, original),
+			fileExpect(p, original),
+			resValidate(r1),
+			resReversal(r1, &r2), // runs in Init to snapshot
+			func() error { // random test
+				// state should be unspecified
+				if st := r2.(*FileRes).State; st == "absent" || st == "exists" {
+					return fmt.Errorf("unexpected state: %s", st)
+				}
+				return nil
+			},
+			resInit(r1),
+			resCheckApply(r1, false), // changed
+			fileExpect(p, content),
+			resCheckApply(r1, true), // it's already good
+			resClose(r1),
+			//resValidate(r2),
+			func() error {
+				// wrap it b/c it is currently nil
+				return r2.Validate()
+			},
+			func() error {
+				return resInit(r2)()
+			},
+			func() error {
+				return resCheckApply(r2, false)()
+			},
+			func() error {
+				return resCheckApply(r2, true)()
+			},
+			func() error {
+				return resClose(r2)()
+			},
+			fileExpect(p, original), // we restored the contents!
+			fileRemove(p),           // cleanup
+		}
+
+		testCases = append(testCases, test{
+			name:     "some file restore",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return nil },
+		})
+	}
+	{
+		//file "/tmp/somefile" {
+		//	content => "some new text\n",
+		//
+		//	Meta:reverse => true,
+		//}
+		//# and there's NO existing file at this path...
+		//# NOTE: This used to be a corner case subtlety for reversal.
+		//# Now that we error in this scenario before reversal, it's ok!
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somefile"
+		res.Path = p
+		//res.State = "exists" // unspecified
+		content := "some new text\n"
+		res.Content = &content
+		var r2 engine.Res // future reversed resource
+
+		timeline := []func() error{
+			fileRemove(p), // ensure no file exists
+			resValidate(r1),
+			resReversal(r1, &r2), // runs in Init to snapshot
+			func() error { // random test
+				// state should be unspecified i think
+				// TODO: or should it be absent?
+				if st := r2.(*FileRes).State; st == "absent" || st == "exists" {
+					return fmt.Errorf("unexpected state: %s", st)
+				}
+				return nil
+			},
+			resInit(r1),
+			resCheckApplyError(r1, false, ErrIsNotExistOK), // changed
+			//fileExpect(p, content),
+			//resCheckApply(r1, true), // it's already good
+			resClose(r1),
+			//func() error {
+			//	// wrap it b/c it is currently nil
+			//	return r2.Validate()
+			//},
+			//func() error {
+			//	return resInit(r2)()
+			//},
+			//func() error { // it's already in the correct state
+			//	return resCheckApply(r2, true)()
+			//},
+			//func() error {
+			//	return resClose(r2)()
+			//},
+			//fileExpect(p, content), // we never changed it back...
+			//fileRemove(p),          // cleanup
+		}
+
+		testCases = append(testCases, test{
+			name:     "ambiguous file restore",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return nil },
+		})
+	}
+	{
+		//file "/tmp/somefile" {
+		//	state => "absent",
+		//
+		//	Meta:reverse => true,
+		//}
+		r1 := makeRes("file", "r1")
+		res := r1.(*FileRes) // if this panics, the test will panic
+		p := "/tmp/somefile"
+		res.Path = p
+		res.State = "absent"
+		original := "this is the original state\n" // original state
+		var r2 engine.Res                          // future reversed resource
+
+		timeline := []func() error{
+			fileWrite(p, original),
+			fileExpect(p, original),
+			resValidate(r1),
+			resReversal(r1, &r2), // runs in Init to snapshot
+			func() error { // random test
+				if st := r2.(*FileRes).State; st != "exists" {
+					return fmt.Errorf("unexpected state: %s", st)
+				}
+				return nil
+			},
+			resInit(r1),
+			resCheckApply(r1, false), // changed
+			fileAbsent(p),            // ensure it got removed
+			resCheckApply(r1, true),  // it's already good
+			resClose(r1),
+			//resValidate(r2), // no!!!
+			func() error {
+				// wrap it b/c it is currently nil
+				return r2.Validate()
+			},
+			func() error {
+				return resInit(r2)()
+			},
+			func() error {
+				return resCheckApply(r2, false)()
+			},
+			func() error {
+				return resCheckApply(r2, true)()
+			},
+			func() error {
+				return resClose(r2)()
+			},
+			fileExpect(p, original), // ensure it's back to original
+		}
+
+		testCases = append(testCases, test{
+			name:     "some removal",
+			timeline: timeline,
+			expect:   func() error { return nil },
+			startup:  func() error { return nil },
+			cleanup:  func() error { return nil },
+		})
+	}
+
+	names := []string{}
+	for index, tc := range testCases { // run all the tests
+		if tc.name == "" {
+			t.Errorf("test #%d: not named", index)
+			continue
+		}
+		if util.StrInList(tc.name, names) {
+			t.Errorf("test #%d: duplicate sub test name of: %s", index, tc.name)
+			continue
+		}
+		names = append(names, tc.name)
+		t.Run(fmt.Sprintf("test #%d (%s)", index, tc.name), func(t *testing.T) {
+			timeline, expect, startup, cleanup := tc.timeline, tc.expect, tc.startup, tc.cleanup
+
+			t.Logf("test #%d: starting...\n", index)
+			defer t.Logf("test #%d: done!", index)
+
+			//debug := testing.Verbose() // set via the -test.v flag to `go test`
+			//logf := func(format string, v ...interface{}) {
+			//	t.Logf(fmt.Sprintf("test #%d: ", index)+format, v...)
+			//}
+
+			t.Logf("test #%d: running startup()", index)
+			if err := startup(); err != nil {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: could not startup: %+v", index, err)
+			}
+			defer func() {
+				t.Logf("test #%d: running cleanup()", index)
+				if err := cleanup(); err != nil {
+					t.Errorf("test #%d: FAIL", index)
+					t.Errorf("test #%d: could not cleanup: %+v", index, err)
+				}
+			}()
+
+			// run timeline
+			t.Logf("test #%d: executing timeline", index)
+			for ix, step := range timeline {
+				t.Logf("test #%d: step(%d)...", index, ix)
+				if err := step(); err != nil {
+					t.Errorf("test #%d: FAIL", index)
+					t.Errorf("test #%d: step(%d) action failed: %s", index, ix, err.Error())
+					break
+				}
+			}
+
+			t.Logf("test #%d: shutting down...", index)
+
+			if err := expect(); err != nil {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: expect failed: %s", index, err.Error())
+				return
+			}
+
+			// all done!
+		})
+	}
+}

engine/resources/svc.go

@@ -354,31 +354,23 @@ func (obj *SvcRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *SvcRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *SvcRes) Compare(r engine.Res) bool {
 	// we can only compare SvcRes to others of the same resource kind
 	res, ok := r.(*SvcRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.State != res.State {
-		return false
+		return fmt.Errorf("the State differs")
 	}
 	if obj.Startup != res.Startup {
-		return false
+		return fmt.Errorf("the Startup differs")
 	}
 	if obj.Session != res.Session {
-		return false
+		return fmt.Errorf("the Session differs")
 	}
 
-	return true
+	return nil
 }
 
 // SvcUID is the UID struct for SvcRes.

engine/resources/test.go

@@ -199,25 +199,17 @@ func (obj *TestRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *TestRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *TestRes) Compare(r engine.Res) bool {
 	// we can only compare TestRes to others of the same resource kind
 	res, ok := r.(*TestRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 	//if obj.Name != res.Name {
 	//	return false
 	//}
 
 	if obj.CompareFail || res.CompareFail {
-		return false
+		return fmt.Errorf("the CompareFail is true")
 	}
 
 	// TODO: yes, I know the long manual version is absurd, but I couldn't
@@ -228,145 +220,145 @@ func (obj *TestRes) Compare(r engine.Res) bool {
 	//}
 
 	if obj.Bool != res.Bool {
-		return false
+		return fmt.Errorf("the Bool differs")
 	}
 	if obj.Str != res.Str {
-		return false
+		return fmt.Errorf("the Str differs")
 	}
 
 	if obj.Int != res.Int {
-		return false
+		return fmt.Errorf("the Str differs")
 	}
 	if obj.Int8 != res.Int8 {
-		return false
+		return fmt.Errorf("the Int8 differs")
 	}
 	if obj.Int16 != res.Int16 {
-		return false
+		return fmt.Errorf("the Int16 differs")
 	}
 	if obj.Int32 != res.Int32 {
-		return false
+		return fmt.Errorf("the Int32 differs")
 	}
 	if obj.Int64 != res.Int64 {
-		return false
+		return fmt.Errorf("the Int64 differs")
 	}
 
 	if obj.Uint != res.Uint {
-		return false
+		return fmt.Errorf("the Uint differs")
 	}
 	if obj.Uint8 != res.Uint8 {
-		return false
+		return fmt.Errorf("the Uint8 differs")
 	}
 	if obj.Uint16 != res.Uint16 {
-		return false
+		return fmt.Errorf("the Uint16 differs")
 	}
 	if obj.Uint32 != res.Uint32 {
-		return false
+		return fmt.Errorf("the Uint32 differs")
 	}
 	if obj.Uint64 != res.Uint64 {
-		return false
+		return fmt.Errorf("the Uint64 differs")
 	}
 
 	//if obj.Uintptr
 	if obj.Byte != res.Byte {
-		return false
+		return fmt.Errorf("the Byte differs")
 	}
 	if obj.Rune != res.Rune {
-		return false
+		return fmt.Errorf("the Rune differs")
 	}
 
 	if obj.Float32 != res.Float32 {
-		return false
+		return fmt.Errorf("the Float32 differs")
 	}
 	if obj.Float64 != res.Float64 {
-		return false
+		return fmt.Errorf("the Float64 differs")
 	}
 	if obj.Complex64 != res.Complex64 {
-		return false
+		return fmt.Errorf("the Complex64 differs")
 	}
 	if obj.Complex128 != res.Complex128 {
-		return false
+		return fmt.Errorf("the Complex128 differs")
 	}
 
 	if (obj.BoolPtr == nil) != (res.BoolPtr == nil) { // xor
-		return false
+		return fmt.Errorf("the BoolPtr differs")
 	}
 	if obj.BoolPtr != nil && res.BoolPtr != nil {
 		if *obj.BoolPtr != *res.BoolPtr { // compare
-			return false
+			return fmt.Errorf("the BoolPtr differs")
 		}
 	}
 	if (obj.StringPtr == nil) != (res.StringPtr == nil) { // xor
-		return false
+		return fmt.Errorf("the StringPtr differs")
 	}
 	if obj.StringPtr != nil && res.StringPtr != nil {
 		if *obj.StringPtr != *res.StringPtr { // compare
-			return false
+			return fmt.Errorf("the StringPtr differs")
 		}
 	}
 	if (obj.Int64Ptr == nil) != (res.Int64Ptr == nil) { // xor
-		return false
+		return fmt.Errorf("the Int64Ptr differs")
 	}
 	if obj.Int64Ptr != nil && res.Int64Ptr != nil {
 		if *obj.Int64Ptr != *res.Int64Ptr { // compare
-			return false
+			return fmt.Errorf("the Int64Ptr differs")
 		}
 	}
 	if (obj.Int8Ptr == nil) != (res.Int8Ptr == nil) { // xor
-		return false
+		return fmt.Errorf("the Int8Ptr differs")
 	}
 	if obj.Int8Ptr != nil && res.Int8Ptr != nil {
 		if *obj.Int8Ptr != *res.Int8Ptr { // compare
-			return false
+			return fmt.Errorf("the Int8Ptr differs")
 		}
 	}
 	if (obj.Uint8Ptr == nil) != (res.Uint8Ptr == nil) { // xor
-		return false
+		return fmt.Errorf("the Uint8Ptr differs")
 	}
 	if obj.Uint8Ptr != nil && res.Uint8Ptr != nil {
 		if *obj.Uint8Ptr != *res.Uint8Ptr { // compare
-			return false
+			return fmt.Errorf("the Uint8Ptr differs")
 		}
 	}
 
 	if !reflect.DeepEqual(obj.Int8PtrPtrPtr, res.Int8PtrPtrPtr) {
-		return false
+		return fmt.Errorf("the Int8PtrPtrPtr differs")
 	}
 
 	if !reflect.DeepEqual(obj.SliceString, res.SliceString) {
-		return false
+		return fmt.Errorf("the SliceString differs")
 	}
 	if !reflect.DeepEqual(obj.MapIntFloat, res.MapIntFloat) {
-		return false
+		return fmt.Errorf("the MapIntFloat differs")
 	}
 	if !reflect.DeepEqual(obj.MixedStruct, res.MixedStruct) {
-		return false
+		return fmt.Errorf("the MixedStruct differs")
 	}
 	if !reflect.DeepEqual(obj.Interface, res.Interface) {
-		return false
+		return fmt.Errorf("the Interface differs")
 	}
 
 	if obj.AnotherStr != res.AnotherStr {
-		return false
+		return fmt.Errorf("the AnotherStr differs")
 	}
 
 	if obj.ValidateBool != res.ValidateBool {
-		return false
+		return fmt.Errorf("the ValidateBool differs")
 	}
 	if obj.ValidateError != res.ValidateError {
-		return false
+		return fmt.Errorf("the ValidateError differs")
 	}
 	if obj.AlwaysGroup != res.AlwaysGroup {
-		return false
+		return fmt.Errorf("the AlwaysGroup differs")
 	}
 	if obj.SendValue != res.SendValue {
-		return false
+		return fmt.Errorf("the SendValue differs")
 	}
 
 	if obj.Comment != res.Comment {
-		return false
+		return fmt.Errorf("the Comment differs")
 	}
 
-	return true
+	return nil
 }
 
 // TestUID is the UID struct for TestRes.

engine/resources/timer.go

@@ -113,25 +113,17 @@ func (obj *TimerRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *TimerRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *TimerRes) Compare(r engine.Res) bool {
 	// we can only compare TimerRes to others of the same resource kind
 	res, ok := r.(*TimerRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.Interval != res.Interval {
-		return false
+		return fmt.Errorf("the Interval differs")
 	}
 
-	return true
+	return nil
 }
 
 // TimerUID is the UID struct for TimerRes.

engine/resources/user.go

@@ -273,45 +273,37 @@ func (obj *UserRes) CheckApply(apply bool) (bool, error) {
 
 // Cmp compares two resources and returns an error if they are not equivalent.
 func (obj *UserRes) Cmp(r engine.Res) error {
-	if !obj.Compare(r) {
-		return fmt.Errorf("did not compare")
-	}
-	return nil
-}
-
-// Compare two resources and return if they are equivalent.
-func (obj *UserRes) Compare(r engine.Res) bool {
 	// we can only compare UserRes to others of the same resource kind
 	res, ok := r.(*UserRes)
 	if !ok {
-		return false
+		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
 	if obj.State != res.State {
-		return false
+		return fmt.Errorf("the State differs")
 	}
 	if (obj.UID == nil) != (res.UID == nil) {
-		return false
+		return fmt.Errorf("the UID differs")
 	}
 	if obj.UID != nil && res.UID != nil {
 		if *obj.UID != *res.UID {
-			return false
+			return fmt.Errorf("the UID differs")
 		}
 	}
 	if (obj.GID == nil) != (res.GID == nil) {
-		return false
+		return fmt.Errorf("the GID differs")
 	}
 	if obj.GID != nil && res.GID != nil {
 		if *obj.GID != *res.GID {
-			return false
+			return fmt.Errorf("the GID differs")
 		}
 	}
 	if (obj.Groups == nil) != (res.Groups == nil) {
-		return false
+		return fmt.Errorf("the Group differs")
 	}
 	if obj.Groups != nil && res.Groups != nil {
 		if len(obj.Groups) != len(res.Groups) {
-			return false
+			return fmt.Errorf("the Group differs")
 		}
 		objGroups := obj.Groups
 		resGroups := res.Groups
@@ -319,22 +311,22 @@ func (obj *UserRes) Compare(r engine.Res) bool {
 		sort.Strings(resGroups)
 		for i := range objGroups {
 			if objGroups[i] != resGroups[i] {
-				return false
+				return fmt.Errorf("the Group differs at index: %d", i)
 			}
 		}
 	}
 	if (obj.HomeDir == nil) != (res.HomeDir == nil) {
-		return false
+		return fmt.Errorf("the HomeDirs differs")
 	}
 	if obj.HomeDir != nil && res.HomeDir != nil {
-		if *obj.HomeDir != *obj.HomeDir {
-			return false
+		if *obj.HomeDir != *res.HomeDir {
+			return fmt.Errorf("the HomeDir differs")
 		}
 	}
 	if obj.AllowDuplicateUID != res.AllowDuplicateUID {
-		return false
+		return fmt.Errorf("the AllowDuplicateUID differs")
 	}
-	return true
+	return nil
 }
 
 // UserUID is the UID struct for UserRes.

engine/reverse.go

@@ -41,10 +41,14 @@ type ReversibleRes interface {
 
 	// Reversed returns the "reverse" or "reciprocal" resource. This is used
 	// to "clean" up after a previously defined resource has been removed.
-	// Interestingly, this returns the core Res interface instead of a
+	// Interestingly, this could return the core Res interface instead of a
 	// ReversibleRes, because there is no requirement that the reverse of a
 	// Res be the same kind of Res, and the reverse might not be reversible!
-	Reversed() (Res, error)
+	// However, in practice, it's nice to use some of the Reversible meta
+	// params in the built value, so keep things simple and have this be a
+	// reversible res. The Res itself doesn't have to implement Reversed()
+	// in a meaningful way, it can just return nil and it will get ignored.
+	Reversed() (ReversibleRes, error)
 }
 
 // ReversibleMeta provides some parameters specific to reversible resources.
@@ -53,6 +57,16 @@ type ReversibleMeta struct {
 	// resource.
 	Disabled bool
 
+	// Reversal specifies that the resource was built from a reversal. This
+	// must be set if the resource was built by a reversal.
+	Reversal bool
+
+	// Overwrite specifies that we should overwrite any existing stored
+	// reversible resource if one that is pending already exists. If this is
+	// false, and a resource with the same name and kind exists, then this
+	// will cause an error.
+	Overwrite bool
+
 	// TODO: add options here, including whether to reverse edges, etc...
 }
 
@@ -61,5 +75,11 @@ func (obj *ReversibleMeta) Cmp(rm *ReversibleMeta) error {
 	if obj.Disabled != rm.Disabled {
 		return fmt.Errorf("values for Disabled are different")
 	}
+	if obj.Reversal != rm.Reversal { // TODO: do we want to compare these?
+		return fmt.Errorf("values for Reversal are different")
+	}
+	if obj.Overwrite != rm.Overwrite {
+		return fmt.Errorf("values for Overwrite are different")
+	}
 	return nil
 }

engine/util/util.go

@@ -23,6 +23,7 @@ import (
 	"encoding/base64"
 	"encoding/gob"
 	"fmt"
+	"os"
 	"os/user"
 	"reflect"
 	"strconv"
@@ -62,6 +63,23 @@ const (
 	DBusSignalJobRemoved = "JobRemoved"
 )
 
+// ResPathUID returns a unique resource UID based on its name and kind. It's
+// safe to use as a token in a path, and as a result has no slashes in it.
+func ResPathUID(res engine.Res) string {
+	// res.Name() is NOT sufficiently unique to use as a UID here, because:
+	// a name of: /tmp/mgmt/foo is /tmp-mgmt-foo and
+	// a name of: /tmp/mgmt-foo -> /tmp-mgmt-foo if we replace slashes.
+	// As a result, we base64 encode (but without slashes).
+	name := strings.Replace(res.Name(), "/", "-", -1) // TODO: use ReplaceAll in 1.12
+	if os.PathSeparator != '/' {                      // lol windows?
+		name = strings.Replace(name, string(os.PathSeparator), "-", -1) // TODO: use ReplaceAll in 1.12
+	}
+	b := []byte(res.Name())
+	encoded := base64.URLEncoding.EncodeToString(b)
+	// Add the safe name on so that it's easier to identify by name...
+	return fmt.Sprintf("%s-%s+%s", res.Kind(), encoded, name)
+}
+
 // ResToB64 encodes a resource to a base64 encoded string (after serialization).
 func ResToB64(res engine.Res) (string, error) {
 	b := bytes.Buffer{}

etcd/etcd.go

@@ -400,7 +400,7 @@ func (obj *EmbdEtcd) Validate() error {
 
 	if obj.NoNetwork {
 		if len(obj.Seeds) != 0 || len(obj.ClientURLs) != 0 || len(obj.ServerURLs) != 0 {
-			return fmt.Errorf("NoNetwork is mutually exclusive with Seeds, ClientURLs and ServerURLs")
+			return fmt.Errorf("option NoNetwork is mutually exclusive with Seeds, ClientURLs and ServerURLs")
 		}
 	}
 

etcd/fs/fs.go

@@ -70,9 +70,9 @@ var (
 	// ErrNotExist is returned when we can't find the requested path.
 	ErrNotExist = os.ErrNotExist
 
-	ErrFileClosed   = errors.New("File is closed")
-	ErrFileReadOnly = errors.New("File handle is read only")
-	ErrOutOfRange   = errors.New("Out of range")
+	ErrFileClosed   = errors.New("file is closed")
+	ErrFileReadOnly = errors.New("file handle is read only")
+	ErrOutOfRange   = errors.New("out of range")
 )
 
 // Fs is a specialized afero.Fs implementation for etcd. It implements a small

etcd/fs/fs_test.go

@@ -231,15 +231,15 @@ func TestFs2(t *testing.T) {
 
 	var memFs = afero.NewMemMapFs()
 
-	if err := util.CopyFs(etcdFs, memFs, "/", "/", false); err != nil {
+	if err := util.CopyFs(etcdFs, memFs, "/", "/", false, false); err != nil {
 		t.Errorf("copyfs error: %+v", err)
 		return
 	}
-	if err := util.CopyFs(etcdFs, memFs, "/", "/", true); err != nil {
+	if err := util.CopyFs(etcdFs, memFs, "/", "/", true, false); err != nil {
 		t.Errorf("copyfs2 error: %+v", err)
 		return
 	}
-	if err := util.CopyFs(etcdFs, memFs, "/", "/tmp/d1/", false); err != nil {
+	if err := util.CopyFs(etcdFs, memFs, "/", "/tmp/d1/", false, false); err != nil {
 		t.Errorf("copyfs3 error: %+v", err)
 		return
 	}
@@ -300,11 +300,11 @@ func TestFs3(t *testing.T) {
 
 	var memFs = afero.NewMemMapFs()
 
-	if err := util.CopyFs(etcdFs, memFs, "/tmp/foo/bar", "/", false); err != nil {
+	if err := util.CopyFs(etcdFs, memFs, "/tmp/foo/bar", "/", false, false); err != nil {
 		t.Errorf("copyfs error: %+v", err)
 		return
 	}
-	if err := util.CopyFs(etcdFs, memFs, "/tmp/foo/bar", "/baz/", false); err != nil {
+	if err := util.CopyFs(etcdFs, memFs, "/tmp/foo/bar", "/baz/", false, false); err != nil {
 		t.Errorf("copyfs2 error: %+v", err)
 		return
 	}
@@ -419,7 +419,7 @@ func TestEtcdCopyFs0(t *testing.T) {
 		t.Logf("tree: \n%s", tree)
 
 		var memFs = afero.NewMemMapFs()
-		if err := util.CopyFs(etcdFs, memFs, tt.cpsrc, tt.cpdst, tt.force); err != nil {
+		if err := util.CopyFs(etcdFs, memFs, tt.cpsrc, tt.cpdst, tt.force, false); err != nil {
 			t.Errorf("copyfs error: %+v", err)
 			return
 		}

examples/lang/autoedges1.mcl

@@ -1,8 +1,9 @@
+$noop = false
 pkg "drbd-utils" {
 	state => "installed",
 
 	Meta:autoedge => true,
-	Meta:noop => true,
+	Meta:noop => $noop,
 }
 
 file "/etc/drbd.conf" {
@@ -10,15 +11,14 @@ file "/etc/drbd.conf" {
 	state => "exists",
 
 	Meta:autoedge => true,
-	Meta:noop => true,
+	Meta:noop => $noop,
 }
 
 file "/etc/drbd.d/" {
-	source => "/dev/null",
 	state => "exists",
 
 	Meta:autoedge => true,
-	Meta:noop => true,
+	Meta:noop => $noop,
 }
 
 # note that the autoedges between the files and the svc don't exist yet :(
@@ -26,5 +26,5 @@ svc "drbd" {
 	state => "stopped",
 
 	Meta:autoedge => true,
-	Meta:noop => true,
+	Meta:noop => $noop,
 }

examples/lang/contains0.mcl

@@ -11,6 +11,7 @@ $c4 = "b" in $set
 $s = fmt.printf("1: %t, 2: %t, 3: %t, 4: %t\n", $c1, $c2, $c3, $c4)
 
 file "/tmp/mgmt/contains" {
+	state => "exists",
 	content => $s,
 }
 
@@ -21,5 +22,6 @@ $x = if sys.hostname() in ["h1", "h3",] {
 }
 
 file "/tmp/mgmt/hello-${sys.hostname()}" {
+	state => "exists",
 	content => $x,
 }

examples/lang/cron2.mcl

@@ -5,4 +5,5 @@ cron "purpleidea-oneshot" {
 
 svc "purpleidea-oneshot" {}
 
+# TODO: do we need a state => "exists" specified here?
 file "/etc/systemd/system/purpleidea-oneshot.service" {}

examples/lang/cron3.mcl

@@ -10,4 +10,5 @@ svc "purpleidea-oneshot" {
 	session => true,
 }
 
+# TODO: do we need a state => "exists" specified here?
 file printf("%s/.config/systemd/user/purpleidea-oneshot.service", $home) {}

examples/lang/datetime1.mcl

@@ -2,5 +2,6 @@ import "datetime"
 
 $d = datetime.now()
 file "/tmp/mgmt/datetime" {
+	state => "exists",
 	content => template("Hello! It is now: {{ datetime_print . }}\n", $d),
 }

examples/lang/datetime2.mcl

@@ -12,6 +12,7 @@ $theload = structlookup(sys.load(), "x1")
 
 if 5 > 3 {
 	file "/tmp/mgmt/datetime" {
+		state => "exists",
 		content => template("Now + 1 year is: {{ .year }} seconds, aka: {{ datetime_print .year }}\n\nload average: {{ .load }}\n", $tmplvalues),
 	}
 }

examples/lang/datetime3.mcl

@@ -14,5 +14,6 @@ $theload = structlookup(sys.load(), "x1")
 $vumeter = example.vumeter("====", 10, 0.9)
 
 file "/tmp/mgmt/datetime" {
+	state => "exists",
 	content => template("Now + 1 year is: {{ .year }} seconds, aka: {{ datetime_print .year }}\n\nload average: {{ .load }}\n\nvu: {{ .vumeter }}\n", $tmplvalues),
 }

examples/lang/env0.mcl

@@ -1,5 +1,5 @@
 # read and print environment variable
-# env TEST=123 EMPTY= ./mgmt run --tmp-prefix --converged-timeout=5 lang --lang=examples/lang/env0.mcl
+# env TEST=123 EMPTY= ./mgmt run --tmp-prefix --converged-timeout=5 lang examples/lang/env0.mcl
 
 import "fmt"
 import "sys"

examples/lang/exchange0.mcl

@@ -4,7 +4,7 @@
 # time ./mgmt run --hostname h2 --seeds http://127.0.0.1:2379 --client-urls http://127.0.0.1:2381 --server-urls http://127.0.0.1:2382 --tmp-prefix --no-pgp empty
 # time ./mgmt run --hostname h3 --seeds http://127.0.0.1:2379 --client-urls http://127.0.0.1:2383 --server-urls http://127.0.0.1:2384 --tmp-prefix --no-pgp empty
 # time ./mgmt run --hostname h4 --seeds http://127.0.0.1:2379 --client-urls http://127.0.0.1:2385 --server-urls http://127.0.0.1:2386 --tmp-prefix --no-pgp empty
-# time ./mgmt deploy --no-git --seeds http://127.0.0.1:2379 lang --lang examples/lang/exchange0.mcl
+# time ./mgmt deploy --no-git --seeds http://127.0.0.1:2379 lang examples/lang/exchange0.mcl
 
 import "sys"
 import "world"
@@ -13,5 +13,6 @@ $rand = random1(8)
 $exchanged = world.exchange("keyns", $rand)
 
 file "/tmp/mgmt/exchange-${sys.hostname()}" {
+	state => "exists",
 	content => template("Found: {{ . }}\n", $exchanged),
 }

examples/lang/history1.mcl

@@ -5,5 +5,6 @@ $dt = datetime.now()
 $hystvalues = {"ix0" => $dt, "ix1" => $dt{1}, "ix2" => $dt{2}, "ix3" => $dt{3},}
 
 file "/tmp/mgmt/history" {
+	state => "exists",
 	content => template("Index(0) {{.ix0}}: {{ datetime_print .ix0 }}\nIndex(1) {{.ix1}}: {{ datetime_print .ix1 }}\nIndex(2) {{.ix2}}: {{ datetime_print .ix2 }}\nIndex(3) {{.ix3}}: {{ datetime_print .ix3 }}\n", $hystvalues),
 }

examples/lang/hysteresis1.mcl

@@ -1,6 +1,7 @@
 import "sys"
 
 file "/tmp/mgmt/systemload" {
+	state => "exists",
 	content => template("load average: {{ .load }} threshold: {{ .threshold }}\n", $tmplvalues),
 }
 

examples/lang/password0.mcl

@@ -3,6 +3,7 @@ password "pass0" {
 }
 
 file "/tmp/mgmt/password" {
+	state => "exists",
 }
 
 Password["pass0"].password -> File["/tmp/mgmt/password"].content

examples/lang/readfile1.mcl

@@ -2,5 +2,6 @@ import "os"
 
 # this copies the contents from /tmp/input and puts them in /tmp/output
 file "/tmp/output" {
+	state => "exists",
 	content => os.readfile("/tmp/input"),
 }

examples/lang/readonlyfriday.mcl

@@ -0,0 +1,30 @@
+import "datetime"
+import "fmt"
+
+$now = datetime.now()
+$day = datetime.weekday($now)
+$is_friday = $day == "friday"
+
+$s1 = template("Hello! It is now: {{ datetime_print . }}\n", $now)
+$s2 = if $is_friday {
+	"It's friday!!! (don't break anything, read-only)"
+} else {
+	if $day == "saturday" || $day == "sunday" {
+		"It's the weekend!"
+	} else {
+		fmt.printf("Unfortunately, it is %s. Go to work!", $day)
+	}
+}
+
+print "msg" {
+	msg => $s1 + $s2,
+}
+
+file "/tmp/files/" {
+	state => "exists",
+	mode => if $is_friday { # this updates the mode, the instant it changes!
+		"0550"
+	} else {
+		"0770"
+	},
+}

examples/lang/regexp0.mcl

@@ -2,8 +2,8 @@ import "fmt"
 import "regexp"
 
 # test with:
-# ./mgmt run --hostname foo.example.com --tmp-prefix lang --lang examples/lang/regexp0.mcl
-# ./mgmt run --hostname db1.example.com --tmp-prefix lang --lang examples/lang/regexp0.mcl
+# ./mgmt run --hostname foo.example.com --tmp-prefix lang examples/lang/regexp0.mcl
+# ./mgmt run --hostname db1.example.com --tmp-prefix lang examples/lang/regexp0.mcl
 print "regexp" {
 	# TODO: add a heredoc string to avoid needing to escape the \ chars
 	msg => fmt.printf("match: %t", regexp.match("^db\\d+\\.example\\.com$", $hostname)),

examples/lang/reverse1.mcl

@@ -0,0 +1,25 @@
+import "datetime"
+import "math"
+
+$now = datetime.now()
+
+# alternate every four seconds
+$mod0 = math.mod($now, 8) == 0
+$mod1 = math.mod($now, 8) == 1
+$mod2 = math.mod($now, 8) == 2
+$mod3 = math.mod($now, 8) == 3
+$mod = $mod0 || $mod1 || $mod2 || $mod3
+
+file "/tmp/mgmt/" {
+	state => "exists",
+}
+
+# file should disappear and re-appear every four seconds
+if $mod {
+	file "/tmp/mgmt/hello" {
+		content => "please say abracadabra...\n",
+		state => "exists",
+
+		Meta:reverse => true,
+	}
+}

examples/lang/reverse2.mcl

@@ -0,0 +1,25 @@
+import "datetime"
+import "math"
+
+$now = datetime.now()
+
+# alternate every four seconds
+$mod0 = math.mod($now, 8) == 0
+$mod1 = math.mod($now, 8) == 1
+$mod2 = math.mod($now, 8) == 2
+$mod3 = math.mod($now, 8) == 3
+$mod = $mod0 || $mod1 || $mod2 || $mod3
+
+file "/tmp/mgmt/" {
+	state => "exists",
+}
+
+# file should re-appear and disappear every four seconds
+# it will even preserve and then restore the pre-existing content!
+if $mod {
+	file "/tmp/mgmt/hello" {
+		state => "absent", # delete the file
+
+		Meta:reverse => true,
+	}
+}

examples/lang/reverse3.mcl

@@ -0,0 +1,26 @@
+import "datetime"
+import "math"
+
+$now = datetime.now()
+
+# alternate every four seconds
+$mod0 = math.mod($now, 8) == 0
+$mod1 = math.mod($now, 8) == 1
+$mod2 = math.mod($now, 8) == 2
+$mod3 = math.mod($now, 8) == 3
+$mod = $mod0 || $mod1 || $mod2 || $mod3
+
+file "/tmp/mgmt/" {
+	state => "exists",
+}
+
+# file should change the mode every four seconds
+# editing the file contents at anytime is allowed
+if $mod {
+	file "/tmp/mgmt/hello" {
+		state => "exists",
+		mode => "0777",
+
+		Meta:reverse => true,
+	}
+}

examples/lang/schedule0.mcl

@@ -17,5 +17,6 @@ $set = world.schedule("xsched", $opts)
 #$set = world.schedule("xsched")
 
 file "/tmp/mgmt/scheduled-${sys.hostname()}" {
+	state => "exists",
 	content => template("set: {{ . }}\n", $set),
 }

examples/lang/sendrecv1.mcl

@@ -19,6 +19,7 @@ Exec["exec0"].output -> Kv["kv0"].value
 
 if $state != "default" {
 	file "/tmp/mgmt/state" {
+		state => "exists",
 		content => fmt.printf("state: %s\n", $state),
 	}
 }

examples/lang/states0.mcl

@@ -7,6 +7,7 @@ $state = maplookup($exchanged, $hostname, "default")
 if $state == "one" || $state == "default" {
 
 	file "/tmp/mgmt/state" {
+		state => "exists",
 		content => "state: one\n",
 	}
 
@@ -22,6 +23,7 @@ if $state == "one" || $state == "default" {
 if $state == "two" {
 
 	file "/tmp/mgmt/state" {
+		state => "exists",
 		content => "state: two\n",
 	}
 
@@ -37,6 +39,7 @@ if $state == "two" {
 if $state == "three" {
 
 	file "/tmp/mgmt/state" {
+		state => "exists",
 		content => "state: three\n",
 	}
 

examples/lang/unicode.mcl

@@ -0,0 +1,8 @@
+$unicode = "ᴊᴀᴍᴇꜱ is cool ⇨ and so is π ☻"
+print "unicode" {
+	msg => $unicode,
+}
+file "/tmp/unicode" {
+	state => "exists",
+	content => $unicode + "\n",
+}

examples/lang/virt2.mcl

@@ -17,6 +17,7 @@ $count = if $input > 8 {
 }
 
 file "/tmp/output" {
+	state => "exists",
 	content => fmt.printf("requesting: %d cpus\n", $count),
 }
 

gapi/helpers.go

@@ -58,10 +58,20 @@ func CopyStringToFs(fs engine.Fs, str, dst string) error {
 }
 
 // CopyDirToFs copies a dir from src path on the local fs to a dst path on fs.
+// FIXME: I'm not sure this does the logical thing when the dst path is a dir.
+// FIXME: We've got a workaround for this inside of the lang CLI GAPI.
 func CopyDirToFs(fs engine.Fs, src, dst string) error {
 	return util.CopyDiskToFs(fs, src, dst, false)
 }
 
+// CopyDirToFsForceAll copies a dir from src path on the local fs to a dst path
+// on fs, but it doesn't error when making a dir that already exists. It also
+// uses `MkdirAll` to prevent some issues.
+// FIXME: This is being added because of issues with CopyDirToFs. POSIX is hard.
+func CopyDirToFsForceAll(fs engine.Fs, src, dst string) error {
+	return util.CopyDiskToFsAll(fs, src, dst, true, true)
+}
+
 // CopyDirContentsToFs copies a dir contents from src path on the local fs to a
 // dst path on fs.
 func CopyDirContentsToFs(fs engine.Fs, src, dst string) error {

integration/instance.go

@@ -373,7 +373,7 @@ func (obj *Instance) DeployLang(code string) error {
 		"deploy", // mode
 		"--no-git",
 		"--seeds", obj.clientURL,
-		"lang", "--lang", filename,
+		"lang", filename,
 	}
 	obj.Logf("run: %s %s", cmdName, strings.Join(cmdArgs, " "))
 	cmd := exec.Command(cmdName, cmdArgs...)

lang/Makefile

@@ -23,8 +23,11 @@ OLDGOYACC := $(shell go version | grep -E 'go1.6|go1.7')
 all: build
 
 build: lexer.nn.go y.go
+	@# recursively run make in child dir named types
+	@$(MAKE) --quiet -C types
 
 clean:
+	$(MAKE) --quiet -C types clean
 	@rm -f lexer.nn.go y.go y.output || true
 
 lexer.nn.go: lexer.nex

lang/funcs/Makefile

@@ -33,6 +33,7 @@ build: $(GENERATED)
 
 # add more input files as dependencies at the end here...
 $(GENERATED): $(MCL_FILES)
+	@echo "Generating: native mcl..."
 	@# go-bindata --pkg bindata -o <OUTPUT> <INPUT>
 	go-bindata --pkg bindata -o ./$@ $^
 	@# gofmt the output file

lang/funcs/contains_polyfunc.go

@@ -48,6 +48,15 @@ type ContainsPolyFunc struct {
 	closeChan chan struct{}
 }
 
+// ArgGen returns the Nth arg name for this function.
+func (obj *ContainsPolyFunc) ArgGen(index int) (string, error) {
+	seq := []string{"needle", "haystack"}
+	if l := len(seq); index >= l {
+		return "", fmt.Errorf("index %d exceeds arg length of %d", index, l)
+	}
+	return seq[index], nil
+}
+
 // Polymorphisms returns the list of possible function signatures available for
 // this static polymorphic function. It relies on type and value hints to limit
 // the number of returned possibilities.
@@ -155,11 +164,15 @@ func (obj *ContainsPolyFunc) Validate() error {
 // Info returns some static info about itself. Build must be called before this
 // will return correct data.
 func (obj *ContainsPolyFunc) Info() *interfaces.Info {
-	typ := types.NewType(fmt.Sprintf("func(needle %s, haystack []%s) bool", obj.Type.String(), obj.Type.String()))
+	var sig *types.Type
+	if obj.Type != nil { // don't panic if called speculatively
+		s := obj.Type.String()
+		sig = types.NewType(fmt.Sprintf("func(needle %s, haystack []%s) bool", s, s))
+	}
 	return &interfaces.Info{
 		Pure: true,
 		Memo: false,
-		Sig:  typ, // func kind
+		Sig:  sig, // func kind
 		Err:  obj.Validate(),
 	}
 }

lang/funcs/core/core.go

@@ -20,7 +20,9 @@ package core
 import (
 	// import so the funcs register
 	_ "github.com/purpleidea/mgmt/lang/funcs/core/datetime"
+	_ "github.com/purpleidea/mgmt/lang/funcs/core/deploy"
 	_ "github.com/purpleidea/mgmt/lang/funcs/core/example"
+	_ "github.com/purpleidea/mgmt/lang/funcs/core/example/nested"
 	_ "github.com/purpleidea/mgmt/lang/funcs/core/fmt"
 	_ "github.com/purpleidea/mgmt/lang/funcs/core/math"
 	_ "github.com/purpleidea/mgmt/lang/funcs/core/os"

lang/funcs/core/core_test.go

@@ -143,29 +143,31 @@ func TestPureFuncExec0(t *testing.T) {
 				return
 			}
 
-			if !reflect.DeepEqual(result, expect) {
-				// double check because DeepEqual is different since the func exists
-				diff := pretty.Compare(result, expect)
-				if diff != "" { // bonus
-					t.Errorf("test #%d: result did not match expected", index)
-					// TODO: consider making our own recursive print function
-					t.Logf("test #%d:   actual: \n\n%s\n", index, spew.Sdump(result))
-					t.Logf("test #%d: expected: \n\n%s", index, spew.Sdump(expect))
-
-					// more details, for tricky cases:
-					diffable := &pretty.Config{
-						Diffable:          true,
-						IncludeUnexported: true,
-						//PrintStringers: false,
-						//PrintTextMarshalers: false,
-						//SkipZeroFields: false,
-					}
-					t.Logf("test #%d:   actual: \n\n%s\n", index, diffable.Sprint(result))
-					t.Logf("test #%d: expected: \n\n%s", index, diffable.Sprint(expect))
-					t.Logf("test #%d: diff:\n%s", index, diff)
-					return
-				}
+			if reflect.DeepEqual(result, expect) {
+				return
 			}
+
+			// double check because DeepEqual is different since the func exists
+			diff := pretty.Compare(result, expect)
+			if diff == "" { // bonus
+				return
+			}
+			t.Errorf("test #%d: result did not match expected", index)
+			// TODO: consider making our own recursive print function
+			t.Logf("test #%d:   actual: \n\n%s\n", index, spew.Sdump(result))
+			t.Logf("test #%d: expected: \n\n%s", index, spew.Sdump(expect))
+
+			// more details, for tricky cases:
+			diffable := &pretty.Config{
+				Diffable:          true,
+				IncludeUnexported: true,
+				//PrintStringers: false,
+				//PrintTextMarshalers: false,
+				//SkipZeroFields: false,
+			}
+			t.Logf("test #%d:   actual: \n\n%s\n", index, diffable.Sprint(result))
+			t.Logf("test #%d: expected: \n\n%s", index, diffable.Sprint(expect))
+			t.Logf("test #%d: diff:\n%s", index, diff)
 		})
 	}
 }

lang/funcs/core/datetime/datetime.go

@@ -18,6 +18,6 @@
 package coredatetime
 
 const (
-	// moduleName is the prefix given to all the functions in this module.
-	moduleName = "datetime"
+	// ModuleName is the prefix given to all the functions in this module.
+	ModuleName = "datetime"
 )

lang/funcs/core/datetime/now_fact.go

@@ -25,7 +25,7 @@ import (
 )
 
 func init() {
-	facts.ModuleRegister(moduleName, "now", func() facts.Fact { return &DateTimeFact{} }) // must register the fact and name
+	facts.ModuleRegister(ModuleName, "now", func() facts.Fact { return &DateTimeFact{} }) // must register the fact and name
 }
 
 // DateTimeFact is a fact which returns the current date and time.

lang/funcs/core/datetime/print_func.go

@@ -27,7 +27,7 @@ import (
 
 func init() {
 	// FIXME: consider renaming this to printf, and add in a format string?
-	simple.ModuleRegister(moduleName, "print", &types.FuncValue{
+	simple.ModuleRegister(ModuleName, "print", &types.FuncValue{
 		T: types.NewType("func(a int) str"),
 		V: func(input []types.Value) (types.Value, error) {
 			epochDelta := input[0].Int()

lang/funcs/core/datetime/weekday_func.go

@@ -0,0 +1,49 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package coredatetime
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/purpleidea/mgmt/lang/funcs/simple"
+	"github.com/purpleidea/mgmt/lang/types"
+)
+
+func init() {
+	simple.ModuleRegister(ModuleName, "weekday", &types.FuncValue{
+		T: types.NewType("func(a int) str"),
+		V: Weekday,
+	})
+}
+
+// Weekday returns the lowercased day of the week corresponding to the input
+// time. The time is the number of seconds since the epoch, and matches what
+// comes from our Now function.
+func Weekday(input []types.Value) (types.Value, error) {
+	epochDelta := input[0].Int()
+	if epochDelta < 0 {
+		return nil, fmt.Errorf("epoch delta must be positive")
+	}
+
+	weekday := time.Unix(epochDelta, 0).Weekday()
+	return &types.StrValue{
+		V: strings.ToLower(weekday.String()),
+	}, nil
+}

lang/funcs/core/deploy/abspath_func.go

@@ -0,0 +1,156 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package coredeploy
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/purpleidea/mgmt/lang/funcs"
+	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/lang/types"
+)
+
+func init() {
+	funcs.ModuleRegister(ModuleName, "abspath", func() interfaces.Func { return &AbsPathFunc{} }) // must register the func and name
+}
+
+const (
+	pathArg = "path"
+)
+
+// AbsPathFunc is a function that returns the absolute, full path in the deploy
+// from an input path that is relative to the calling file. If you pass it an
+// empty string, you'll just get the absolute deploy directory path that you're
+// in.
+type AbsPathFunc struct {
+	init *interfaces.Init
+	data *interfaces.FuncData
+	last types.Value // last value received to use for diff
+
+	path   string // the active path
+	result string // last calculated output
+
+	closeChan chan struct{}
+}
+
+// SetData is used by the language to pass our function some code-level context.
+func (obj *AbsPathFunc) SetData(data *interfaces.FuncData) {
+	obj.data = data
+}
+
+// ArgGen returns the Nth arg name for this function.
+func (obj *AbsPathFunc) ArgGen(index int) (string, error) {
+	seq := []string{pathArg}
+	if l := len(seq); index >= l {
+		return "", fmt.Errorf("index %d exceeds arg length of %d", index, l)
+	}
+	return seq[index], nil
+}
+
+// Validate makes sure we've built our struct properly. It is usually unused for
+// normal functions that users can use directly.
+func (obj *AbsPathFunc) Validate() error {
+	return nil
+}
+
+// Info returns some static info about itself.
+func (obj *AbsPathFunc) Info() *interfaces.Info {
+	return &interfaces.Info{
+		Pure: false, // maybe false because the file contents can change
+		Memo: false,
+		Sig:  types.NewType(fmt.Sprintf("func(%s str) str", pathArg)),
+	}
+}
+
+// Init runs some startup code for this function.
+func (obj *AbsPathFunc) Init(init *interfaces.Init) error {
+	obj.init = init
+	obj.closeChan = make(chan struct{})
+	if obj.data == nil {
+		// programming error
+		return fmt.Errorf("missing function data")
+	}
+	return nil
+}
+
+// Stream returns the changing values that this func has over time.
+func (obj *AbsPathFunc) Stream() error {
+	defer close(obj.init.Output) // the sender closes
+	for {
+		select {
+		case input, ok := <-obj.init.Input:
+			if !ok {
+				obj.init.Input = nil // don't infinite loop back
+				continue             // no more inputs, but don't return!
+			}
+			//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
+			//	return errwrap.Wrapf(err, "wrong function input")
+			//}
+
+			if obj.last != nil && input.Cmp(obj.last) == nil {
+				continue // value didn't change, skip it
+			}
+			obj.last = input // store for next
+
+			path := input.Struct()[pathArg].Str()
+			// TODO: add validation for absolute path?
+			if path == obj.path {
+				continue // nothing changed
+			}
+			obj.path = path
+
+			p := strings.TrimSuffix(obj.data.Base, "/")
+			if p == obj.data.Base { // didn't trim, so we fail
+				// programming error
+				return fmt.Errorf("no trailing slash on Base, got: `%s`", p)
+			}
+			result := p
+
+			if obj.path == "" {
+				result += "/" // add the above trailing slash back
+			} else if !strings.HasPrefix(obj.path, "/") {
+				return fmt.Errorf("path was not absolute, got: `%s`", obj.path)
+				//result += "/" // be forgiving ?
+			}
+			result += obj.path
+
+			if obj.result == result {
+				continue // result didn't change
+			}
+			obj.result = result // store new result
+
+		case <-obj.closeChan:
+			return nil
+		}
+
+		select {
+		case obj.init.Output <- &types.StrValue{
+			V: obj.result,
+		}:
+		case <-obj.closeChan:
+			return nil
+		}
+	}
+}
+
+// Close runs some shutdown code for this function and turns off the stream.
+func (obj *AbsPathFunc) Close() error {
+	close(obj.closeChan)
+	return nil
+}

lang/funcs/core/deploy/deploy.go

@@ -0,0 +1,23 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package coredeploy
+
+const (
+	// ModuleName is the prefix given to all the functions in this module.
+	ModuleName = "deploy"
+)

lang/funcs/core/deploy/readfile_func.go

@@ -0,0 +1,165 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package coredeploy
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/purpleidea/mgmt/lang/funcs"
+	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/lang/types"
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+func init() {
+	funcs.ModuleRegister(ModuleName, "readfile", func() interfaces.Func { return &ReadFileFunc{} }) // must register the func and name
+}
+
+// ReadFileFunc is a function that reads the full contents from a file in our
+// deploy. The file contents can only change with a new deploy, so this is
+// static. Please note that this is different from the readfile function in the
+// os package.
+type ReadFileFunc struct {
+	init *interfaces.Init
+	data *interfaces.FuncData
+	last types.Value // last value received to use for diff
+
+	filename string // the active filename
+	result   string // last calculated output
+
+	closeChan chan struct{}
+}
+
+// SetData is used by the language to pass our function some code-level context.
+func (obj *ReadFileFunc) SetData(data *interfaces.FuncData) {
+	obj.data = data
+}
+
+// ArgGen returns the Nth arg name for this function.
+func (obj *ReadFileFunc) ArgGen(index int) (string, error) {
+	seq := []string{"filename"}
+	if l := len(seq); index >= l {
+		return "", fmt.Errorf("index %d exceeds arg length of %d", index, l)
+	}
+	return seq[index], nil
+}
+
+// Validate makes sure we've built our struct properly. It is usually unused for
+// normal functions that users can use directly.
+func (obj *ReadFileFunc) Validate() error {
+	return nil
+}
+
+// Info returns some static info about itself.
+func (obj *ReadFileFunc) Info() *interfaces.Info {
+	return &interfaces.Info{
+		Pure: false, // maybe false because the file contents can change
+		Memo: false,
+		Sig:  types.NewType("func(filename str) str"),
+	}
+}
+
+// Init runs some startup code for this function.
+func (obj *ReadFileFunc) Init(init *interfaces.Init) error {
+	obj.init = init
+	obj.closeChan = make(chan struct{})
+	if obj.data == nil {
+		// programming error
+		return fmt.Errorf("missing function data")
+	}
+	return nil
+}
+
+// Stream returns the changing values that this func has over time.
+func (obj *ReadFileFunc) Stream() error {
+	defer close(obj.init.Output) // the sender closes
+	for {
+		select {
+		case input, ok := <-obj.init.Input:
+			if !ok {
+				obj.init.Input = nil // don't infinite loop back
+				continue             // no more inputs, but don't return!
+			}
+			//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
+			//	return errwrap.Wrapf(err, "wrong function input")
+			//}
+
+			if obj.last != nil && input.Cmp(obj.last) == nil {
+				continue // value didn't change, skip it
+			}
+			obj.last = input // store for next
+
+			filename := input.Struct()["filename"].Str()
+			// TODO: add validation for absolute path?
+			if filename == obj.filename {
+				continue // nothing changed
+			}
+			obj.filename = filename
+
+			p := strings.TrimSuffix(obj.data.Base, "/")
+			if p == obj.data.Base { // didn't trim, so we fail
+				// programming error
+				return fmt.Errorf("no trailing slash on Base, got: `%s`", p)
+			}
+			path := p
+
+			if !strings.HasPrefix(obj.filename, "/") {
+				return fmt.Errorf("filename was not absolute, got: `%s`", obj.filename)
+				//path += "/" // be forgiving ?
+			}
+			path += obj.filename
+
+			fs, err := obj.init.World.Fs(obj.data.FsURI) // open the remote file system
+			if err != nil {
+				return errwrap.Wrapf(err, "can't load code from file system `%s`", obj.data.FsURI)
+			}
+			// this is relative to the module dir the func is in!
+			content, err := fs.ReadFile(path) // open the remote file system
+			// We could use it directly, but it feels like less correct.
+			//content, err := obj.data.Fs.ReadFile(path) // open the remote file system
+			if err != nil {
+				return errwrap.Wrapf(err, "can't read file `%s` (%s)", obj.filename, path)
+			}
+
+			result := string(content) // convert to string
+
+			if obj.result == result {
+				continue // result didn't change
+			}
+			obj.result = result // store new result
+
+		case <-obj.closeChan:
+			return nil
+		}
+
+		select {
+		case obj.init.Output <- &types.StrValue{
+			V: obj.result,
+		}:
+		case <-obj.closeChan:
+			return nil
+		}
+	}
+}
+
+// Close runs some shutdown code for this function and turns off the stream.
+func (obj *ReadFileFunc) Close() error {
+	close(obj.closeChan)
+	return nil
+}

lang/funcs/core/deploy/readfileabs_func.go

@@ -0,0 +1,151 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package coredeploy
+
+import (
+	"fmt"
+
+	"github.com/purpleidea/mgmt/lang/funcs"
+	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/lang/types"
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+func init() {
+	funcs.ModuleRegister(ModuleName, "readfileabs", func() interfaces.Func { return &ReadFileAbsFunc{} }) // must register the func and name
+}
+
+// ReadFileAbsFunc is a function that reads the full contents from a file in our
+// deploy. The file contents can only change with a new deploy, so this is
+// static. In particular, this takes an absolute path relative to the root
+// deploy. In general, you should use `deploy.readfile` instead. Please note
+// that this is different from the readfile function in the os package.
+type ReadFileAbsFunc struct {
+	init *interfaces.Init
+	data *interfaces.FuncData
+	last types.Value // last value received to use for diff
+
+	filename string // the active filename
+	result   string // last calculated output
+
+	closeChan chan struct{}
+}
+
+// SetData is used by the language to pass our function some code-level context.
+func (obj *ReadFileAbsFunc) SetData(data *interfaces.FuncData) {
+	obj.data = data
+}
+
+// ArgGen returns the Nth arg name for this function.
+func (obj *ReadFileAbsFunc) ArgGen(index int) (string, error) {
+	seq := []string{"filename"}
+	if l := len(seq); index >= l {
+		return "", fmt.Errorf("index %d exceeds arg length of %d", index, l)
+	}
+	return seq[index], nil
+}
+
+// Validate makes sure we've built our struct properly. It is usually unused for
+// normal functions that users can use directly.
+func (obj *ReadFileAbsFunc) Validate() error {
+	return nil
+}
+
+// Info returns some static info about itself.
+func (obj *ReadFileAbsFunc) Info() *interfaces.Info {
+	return &interfaces.Info{
+		Pure: false, // maybe false because the file contents can change
+		Memo: false,
+		Sig:  types.NewType("func(filename str) str"),
+	}
+}
+
+// Init runs some startup code for this function.
+func (obj *ReadFileAbsFunc) Init(init *interfaces.Init) error {
+	obj.init = init
+	obj.closeChan = make(chan struct{})
+	if obj.data == nil {
+		// programming error
+		return fmt.Errorf("missing function data")
+	}
+	return nil
+}
+
+// Stream returns the changing values that this func has over time.
+func (obj *ReadFileAbsFunc) Stream() error {
+	defer close(obj.init.Output) // the sender closes
+	for {
+		select {
+		case input, ok := <-obj.init.Input:
+			if !ok {
+				obj.init.Input = nil // don't infinite loop back
+				continue             // no more inputs, but don't return!
+			}
+			//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
+			//	return errwrap.Wrapf(err, "wrong function input")
+			//}
+
+			if obj.last != nil && input.Cmp(obj.last) == nil {
+				continue // value didn't change, skip it
+			}
+			obj.last = input // store for next
+
+			filename := input.Struct()["filename"].Str()
+			// TODO: add validation for absolute path?
+			if filename == obj.filename {
+				continue // nothing changed
+			}
+			obj.filename = filename
+
+			fs, err := obj.init.World.Fs(obj.data.FsURI) // open the remote file system
+			if err != nil {
+				return errwrap.Wrapf(err, "can't load code from file system `%s`", obj.data.FsURI)
+			}
+			content, err := fs.ReadFile(obj.filename) // open the remote file system
+			// We could use it directly, but it feels like less correct.
+			//content, err := obj.data.Fs.ReadFile(obj.filename) // open the remote file system
+			if err != nil {
+				return errwrap.Wrapf(err, "can't read file `%s`", obj.filename)
+			}
+
+			result := string(content) // convert to string
+
+			if obj.result == result {
+				continue // result didn't change
+			}
+			obj.result = result // store new result
+
+		case <-obj.closeChan:
+			return nil
+		}
+
+		select {
+		case obj.init.Output <- &types.StrValue{
+			V: obj.result,
+		}:
+		case <-obj.closeChan:
+			return nil
+		}
+	}
+}
+
+// Close runs some shutdown code for this function and turns off the stream.
+func (obj *ReadFileAbsFunc) Close() error {
+	close(obj.closeChan)
+	return nil
+}

lang/funcs/core/example/answer_func.go

@@ -26,7 +26,7 @@ import (
 const Answer = 42
 
 func init() {
-	simple.ModuleRegister(moduleName, "answer", &types.FuncValue{
+	simple.ModuleRegister(ModuleName, "answer", &types.FuncValue{
 		T: types.NewType("func() int"),
 		V: func([]types.Value) (types.Value, error) {
 			return &types.IntValue{V: Answer}, nil

lang/funcs/core/example/errorbool_func.go

@@ -25,7 +25,7 @@ import (
 )
 
 func init() {
-	simple.ModuleRegister(moduleName, "errorbool", &types.FuncValue{
+	simple.ModuleRegister(ModuleName, "errorbool", &types.FuncValue{
 		T: types.NewType("func(a bool) str"),
 		V: func(input []types.Value) (types.Value, error) {
 			if input[0].Bool() {

lang/funcs/core/example/example.go

@@ -18,6 +18,6 @@
 package coreexample
 
 const (
-	// moduleName is the prefix given to all the functions in this module.
-	moduleName = "example"
+	// ModuleName is the prefix given to all the functions in this module.
+	ModuleName = "example"
 )

lang/funcs/core/example/flipflop_fact.go

@@ -25,7 +25,7 @@ import (
 )
 
 func init() {
-	facts.ModuleRegister(moduleName, "flipflop", func() facts.Fact { return &FlipFlopFact{} }) // must register the fact and name
+	facts.ModuleRegister(ModuleName, "flipflop", func() facts.Fact { return &FlipFlopFact{} }) // must register the fact and name
 }
 
 // FlipFlopFact is a fact which flips a bool repeatedly. This is an example fact

lang/funcs/core/example/int2str_func.go

@@ -25,7 +25,7 @@ import (
 )
 
 func init() {
-	simple.ModuleRegister(moduleName, "int2str", &types.FuncValue{
+	simple.ModuleRegister(ModuleName, "int2str", &types.FuncValue{
 		T: types.NewType("func(a int) str"),
 		V: func(input []types.Value) (types.Value, error) {
 			return &types.StrValue{

lang/funcs/core/example/nativeanswer.mcl

@@ -0,0 +1,21 @@
+# Mgmt
+# Copyright (C) 2013-2019+ James Shubin and the project contributors
+# Written by James Shubin <james@shubin.ca> and the project contributors
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+# This is a native (mcl) function.
+func nativeanswer() {
+	42
+}

lang/funcs/core/example/nested/hello_func.go

@@ -0,0 +1,38 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package corenested
+
+import (
+	coreexample "github.com/purpleidea/mgmt/lang/funcs/core/example"
+	"github.com/purpleidea/mgmt/lang/funcs/simple"
+	"github.com/purpleidea/mgmt/lang/types"
+)
+
+func init() {
+	simple.ModuleRegister(coreexample.ModuleName+"/"+ModuleName, "hello", &types.FuncValue{
+		T: types.NewType("func() str"),
+		V: Hello,
+	})
+}
+
+// Hello returns some string. This is just to test nesting.
+func Hello(input []types.Value) (types.Value, error) {
+	return &types.StrValue{
+		V: "Hello!",
+	}, nil
+}

lang/funcs/core/example/nested/nested.go

@@ -0,0 +1,23 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package corenested
+
+const (
+	// ModuleName is the prefix given to all the functions in this module.
+	ModuleName = "nested"
+)

lang/funcs/core/example/plus_func.go

@@ -0,0 +1,38 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package coreexample
+
+import (
+	"github.com/purpleidea/mgmt/lang/funcs/simple"
+	"github.com/purpleidea/mgmt/lang/types"
+)
+
+func init() {
+	simple.ModuleRegister(ModuleName, "plus", &types.FuncValue{
+		T: types.NewType("func(y str, z str) str"),
+		V: Plus,
+	})
+}
+
+// Plus returns y + z.
+func Plus(input []types.Value) (types.Value, error) {
+	y, z := input[0].Str(), input[1].Str()
+	return &types.StrValue{
+		V: y + z,
+	}, nil
+}

lang/funcs/core/example/str2int_func.go

@@ -25,7 +25,7 @@ import (
 )
 
 func init() {
-	simple.ModuleRegister(moduleName, "str2int", &types.FuncValue{
+	simple.ModuleRegister(ModuleName, "str2int", &types.FuncValue{
 		T: types.NewType("func(a str) int"),
 		V: func(input []types.Value) (types.Value, error) {
 			var i int64

lang/funcs/core/example/test1.mcl

@@ -0,0 +1,31 @@
+# Mgmt
+# Copyright (C) 2013-2019+ James Shubin and the project contributors
+# Written by James Shubin <james@shubin.ca> and the project contributors
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+# This is a native (mcl) function.
+func test1() {
+	42
+}
+
+# This is a native (mcl) global variable.
+$test1 = 13
+
+# This is a native (mcl) class.
+class test1 {
+	print "inside" {
+		msg => "wow, cool",
+	}
+}

lang/funcs/core/example/vumeter_func.go

@@ -33,7 +33,7 @@ import (
 )
 
 func init() {
-	funcs.ModuleRegister(moduleName, "vumeter", func() interfaces.Func { return &VUMeterFunc{} }) // must register the func and name
+	funcs.ModuleRegister(ModuleName, "vumeter", func() interfaces.Func { return &VUMeterFunc{} }) // must register the func and name
 }
 
 // VUMeterFunc is a gimmic function to display a vu meter from the microphone.
@@ -50,6 +50,15 @@ type VUMeterFunc struct {
 	closeChan chan struct{}
 }
 
+// ArgGen returns the Nth arg name for this function.
+func (obj *VUMeterFunc) ArgGen(index int) (string, error) {
+	seq := []string{"symbol", "multiplier", "peak"}
+	if l := len(seq); index >= l {
+		return "", fmt.Errorf("index %d exceeds arg length of %d", index, l)
+	}
+	return seq[index], nil
+}
+
 // Validate makes sure we've built our struct properly. It is usually unused for
 // normal functions that users can use directly.
 func (obj *VUMeterFunc) Validate() error {

lang/funcs/core/fmt/fmt.go

@@ -18,6 +18,6 @@
 package corefmt
 
 const (
-	// moduleName is the prefix given to all the functions in this module.
-	moduleName = "fmt"
+	// ModuleName is the prefix given to all the functions in this module.
+	ModuleName = "fmt"
 )

lang/funcs/core/fmt/printf_func.go

@@ -29,13 +29,11 @@ import (
 
 func init() {
 	// FIXME: should this be named sprintf instead?
-	funcs.ModuleRegister(moduleName, "printf", func() interfaces.Func { return &PrintfFunc{} })
+	funcs.ModuleRegister(ModuleName, "printf", func() interfaces.Func { return &PrintfFunc{} })
 }
 
 const (
-	// XXX: does this need to be `a` ? -- for now yes, fix this compiler bug
-	//formatArgName = "format" // name of the first arg
-	formatArgName = "a" // name of the first arg
+	formatArgName = "format" // name of the first arg
 )
 
 // PrintfFunc is a static polymorphic function that compiles a format string and
@@ -58,6 +56,14 @@ type PrintfFunc struct {
 	closeChan chan struct{}
 }
 
+// ArgGen returns the Nth arg name for this function.
+func (obj *PrintfFunc) ArgGen(index int) (string, error) {
+	if index == 0 {
+		return formatArgName, nil
+	}
+	return util.NumToAlpha(index - 1), nil
+}
+
 // Polymorphisms returns the possible type signature for this function. In this
 // case, since the number of arguments can be infinite, it returns the final
 // precise type if it can be gleamed from the format argument. If it cannot, it
@@ -108,9 +114,9 @@ func (obj *PrintfFunc) Polymorphisms(partialType *types.Type, partialValues []ty
 	typ.Ord = append(typ.Ord, formatArgName)
 
 	for i, x := range typList {
-		name := util.NumToAlpha(i + 1) // +1 to skip the format arg
+		name := util.NumToAlpha(i) // start with a...
 		if name == formatArgName {
-			return nil, fmt.Errorf("could not build function with %d args", i+1)
+			return nil, fmt.Errorf("could not build function with %d args", i+1) // +1 for format arg
 		}
 
 		// if we also had even more partial type information, check it!

lang/funcs/core/math/math.go

@@ -18,6 +18,6 @@
 package coremath
 
 const (
-	// moduleName is the prefix given to all the functions in this module.
-	moduleName = "math"
+	// ModuleName is the prefix given to all the functions in this module.
+	ModuleName = "math"
 )

lang/funcs/core/math/mod_func.go

@@ -0,0 +1,78 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package coremath
+
+import (
+	"fmt"
+	"math"
+
+	"github.com/purpleidea/mgmt/lang/funcs/simplepoly"
+	"github.com/purpleidea/mgmt/lang/types"
+)
+
+func init() {
+	simplepoly.ModuleRegister(ModuleName, "mod", []*types.FuncValue{
+		{
+			T: types.NewType("func(int, int) int"),
+			V: Mod,
+		},
+		{
+			T: types.NewType("func(float, float) float"),
+			V: Mod,
+		},
+	})
+}
+
+// Mod returns mod(x, y), the remainder of x/y. The two values must be either
+// both of KindInt or both of KindFloat, and it will return the same kind. If
+// you pass in a divisor of zero, this will error, eg: mod(x, 0) = NaN.
+// TODO: consider returning zero instead of erroring?
+func Mod(input []types.Value) (types.Value, error) {
+	var x, y float64
+	var float bool
+	k := input[0].Type().Kind
+	switch k {
+	case types.KindFloat:
+		float = true
+		x = input[0].Float()
+		y = input[1].Float()
+	case types.KindInt:
+		x = float64(input[0].Int())
+		y = float64(input[1].Int())
+	default:
+		return nil, fmt.Errorf("unexpected kind: %s", k)
+	}
+	z := math.Mod(x, y)
+	if math.IsNaN(z) {
+		return nil, fmt.Errorf("result is not a number")
+	}
+	if math.IsInf(z, 1) {
+		return nil, fmt.Errorf("unexpected positive infinity")
+	}
+	if math.IsInf(z, -1) {
+		return nil, fmt.Errorf("unexpected negative infinity")
+	}
+	if float {
+		return &types.FloatValue{
+			V: z,
+		}, nil
+	}
+	return &types.IntValue{
+		V: int64(z), // XXX: does this truncate?
+	}, nil
+}

lang/funcs/core/math/pow_func.go

@@ -26,7 +26,7 @@ import (
 )
 
 func init() {
-	simple.ModuleRegister(moduleName, "pow", &types.FuncValue{
+	simple.ModuleRegister(ModuleName, "pow", &types.FuncValue{
 		T: types.NewType("func(x float, y float) float"),
 		V: Pow,
 	})

lang/funcs/core/math/sqrt_func.go

@@ -26,7 +26,7 @@ import (
 )
 
 func init() {
-	simple.ModuleRegister(moduleName, "sqrt", &types.FuncValue{
+	simple.ModuleRegister(ModuleName, "sqrt", &types.FuncValue{
 		T: types.NewType("func(x float) float"),
 		V: Sqrt,
 	})