modules: Add a small stub for CUPS

This is definitely not perfect, but it's a simple stub which we can
expand on.

.github/workflows/test.yaml

@@ -27,9 +27,9 @@ jobs:
           # macos tests are currently failing in CI
           #- macos-latest
         golang_version:
-          # TODO: add 1.21.x and tip
+          # TODO: add 1.24.x and tip
           # minimum required and latest published go_version
-          - "1.20"
+          - "1.23"
         test_block:
           - basic
           - shell

.lycheeignore

@@ -0,0 +1,3 @@
+# list URLs that should be excluded for lychee link checher
+https://roidelapluie.be
+https://github.com/purpleidea/mgmt/commit

.travis.yml

@@ -1,63 +0,0 @@
-language: go
-os:
-  - linux
-go_import_path: github.com/purpleidea/mgmt
-sudo: true
-dist: xenial
-# travis requires that you update manually, and provides this key to trigger it
-apt:
-  update: true
-before_install:
-  # print some debug information to help catch the constant travis regressions
-  - if [ -e /etc/apt/sources.list.d/ ]; then sudo ls -l /etc/apt/sources.list.d/; fi
-  # workaround broken travis NO_PUBKEY errors
-  - if [ -e /etc/apt/sources.list.d/rabbitmq_rabbitmq-server.list ]; then sudo rm -f /etc/apt/sources.list.d/rabbitmq_rabbitmq-server.list; fi
-  - if [ -e /etc/apt/sources.list.d/github_git-lfs.list ]; then sudo rm -f /etc/apt/sources.list.d/github_git-lfs.list; fi
-  # as per a number of comments online, this might mitigate some flaky fails...
-  - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6; fi
-  # apt update tends to be flaky in travis, retry up to 3 times on failure
-  # https://docs.travis-ci.com/user/common-build-problems/#travis_retry
-  - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then travis_retry travis_retry sudo apt update; fi
-  - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
-  - git fetch --unshallow
-install: 'make deps'
-matrix:
-  fast_finish: false
-  allow_failures:
-  - go: 1.21.x
-  - go: tip
-  - os: osx
-  # include only one build for osx for a quicker build as the nr. of these runners are sparse
-  include:
-  - name: "basic tests"
-    go: 1.20.x
-    env: TEST_BLOCK=basic
-  - name: "shell tests"
-    go: 1.20.x
-    env: TEST_BLOCK=shell
-  - name: "race tests"
-    go: 1.20.x
-    env: TEST_BLOCK=race
-  - go: 1.21.x
-  - go: tip
-  - os: osx
-script: 'TEST_BLOCK="$TEST_BLOCK" make test'
-
-# the "secure" channel value is the result of running: ./misc/travis-encrypt.sh
-# with a value of: irc.freenode.net#mgmtconfig to eliminate noise from forks...
-notifications:
-  irc:
-    #channels:
-    #  - secure: htcuWAczm3C1zKC9vUfdRzhIXM1vtF+q0cLlQFXK1IQQlk693/pM30Mmf2L/9V2DVDeps+GyLdip0ARXD1DZEJV0lK+Ca1qbHdFP1r4Xv6l5+jaDb5Y88YU5LI8K758QShiZJojuQ1aO2j8xmmt9V0/5y5QwlpPeHbKYBOFPBX3HvlT9DhvwZNKGhBb4qJOEaPVOwq9IkN3DyQ456MHcJ3q3vF9Lb440uTuLsJNof2AbYZH8ZIHCSG2N8tBj2qhJOpWQboYtQJzE2pRaGkGBL4kYcHZSZMXX8sl4cBM1vx/IRUkvBxJUpLJz2gn/eRI+/gr59juZE2K0+FOLlx9dLnX626Y9xSViopBI6JsIoHJDqNC7aGaF2qaYulGYN65VNKVqmghjgt6JLmmiKeH10hYrJMMvt2rms8l4+5iwmCwXvhH/WU9edzk2p5wqERMnostJFEJib0zI3yzLoF0sdJs+veKtagzfayY2d2l7hlmt951IpqqVWldVgWUcQKVvi8gmRarbwFlK+5D7BEnkUDcLNly/cqf7BgEeX6YfF+FiR4pgfOhYvGCD+2q91NgWQXHBCxbyN0be1TVdkXD94f0Lkn94VyEJJ+PkPlG+rPgFwGcjqN4oEGkJeJmES2If05q2Ms1dJLwYQDL3+Py4lNMSdSWj24TzlFVhtwHepuw=
-    template:
-      - "%{repository} (%{commit}: %{author}): %{message}"
-      - "More info : %{build_url}"
-    on_success: always
-    on_failure: always
-    use_notice: false
-    skip_join: false
-  email:
-    recipients:
-      - secure: qNkgP6QLl6VXpFQIxas2wggxvIiOmm1/hGRXm4BXsSFzHsJPvMamA3E1HEC7H+luiWTny1jtGSGgTJPV9CX1LtQV0g0S4ThaAvWuKvk3rXO8IVd++iA/Lh1s1H6JdKM0dJtLqFICawjeci4tOQzSvrM2eCBWqT0UYsrQsGHB6AF31GNAH0Acqd5cYeL+ZpbCN+hQEznAZQ7546N25TwqieI8Lg7nisA+lwYYwsaC2+f5RIeyvvKjQv3wzEdBAQ9CI9WQiTOUBnUnyYxMrdomQ/XGF66QnZy9vq5nEP83IFtuhPvSamL7ceT+yJW0jDyBi8sYEV7On7eXzjyHbiYpF4YHcJrFnf5RyV4kQGd6/SC8iZwK4Is4eyeAjDFTC+JafLajw9R9x9bK43BwlRAWOZxjFKe0cU/BVAjmlz87vHgUho2P41+0a5XfajfU6VhA5QFPK6rNH7W1CnA7D/0LmS0yaqJM1OCrm6LfoZEMhe0DxTJ9uWJbr0x1sYao6q8H4xYk+fyRgoBAr2TxYU7kXx8ThiRdzuQ8izdbojlzTYLe8liZMIsjL0axLsLK7YBWrjJUcDFDjR/DqmVxPrvbVFbCi9ChmBw0WmbJvDY0FV8T8dO8wCjg9JEmprAmWPyq0g/F87LFK4tAZqQFJGjP1qwsR9jdwdNTKeCdY656f/Y=
-    on_failure: change
-    on_success: change

COPYRIGHT

@@ -1,5 +1,5 @@
 Mgmt
-Copyright (C) 2013-2024+ James Shubin and the project contributors
+Copyright (C) James Shubin and the project contributors
 Written by James Shubin <james@shubin.ca> and the project contributors
 
 This program is free software: you can redistribute it and/or modify

Makefile

@@ -1,5 +1,5 @@
 # Mgmt
-# Copyright (C) 2013-2024+ James Shubin and the project contributors
+# Copyright (C) James Shubin and the project contributors
 # Written by James Shubin <james@shubin.ca> and the project contributors
 #
 # This program is free software: you can redistribute it and/or modify
@@ -225,6 +225,8 @@ build-debug: $(PROGRAM)
 GOOS=$(firstword $(subst -, ,$*))
 GOARCH=$(lastword $(subst -, ,$*))
 build/mgmt-%: $(GO_FILES) $(MCL_FILES) go.mod go.sum | lang funcgen
+	@# If you need to run `go mod tidy` then this can trigger.
+	@if [ "$(PKGNAME)" = "" ]; then echo "\$$(PKGNAME) is empty, test with: go list ."; exit 42; fi
 	@echo "Building: $(PROGRAM), os/arch: $*, version: $(SVERSION)..."
 	time env GOOS=${GOOS} GOARCH=${GOARCH} go build $(TRIMPATH) -ldflags=$(PKGNAME)="-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS)
 

README.md

@@ -10,13 +10,19 @@
 [![Patreon](https://img.shields.io/badge/patreon-donate-yellow.svg?style=flat-square)](https://www.patreon.com/purpleidea)
 [![Liberapay](https://img.shields.io/badge/liberapay-donate-yellow.svg?style=flat-square)](https://liberapay.com/purpleidea/donate)
 
+> [!TIP]
+> [Resource reference guide now available!](https://mgmtconfig.com/docs/resources/)
+
+> [!TIP]
+> [Function reference guide now available!](https://mgmtconfig.com/docs/functions/)
+
 ## About:
 
 `Mgmt` is a real-time automation tool. It is familiar to existing configuration
 management software, but is drastically more powerful as it can allow you to
 build real-time, closed-loop feedback systems, in a very safe way, and with a
-surprisingly small amout of our `mcl` code. For example, the following code will
-ensure that your file server is set to read-only when it's friday.
+surprisingly small amount of our `mcl` code. For example, the following code
+will ensure that your file server is set to read-only when it's friday.
 
 ```mcl
 import "datetime"
@@ -92,12 +98,17 @@ Please read, enjoy and help improve our documentation!
 | [quick start guide](docs/quick-start-guide.md) | for everyone |
 | [frequently asked questions](docs/faq.md) | for everyone |
 | [general documentation](docs/documentation.md) | for everyone |
+| [resource reference](https://mgmtconfig.com/docs/resources/) | for everyone |
+| [function reference](https://mgmtconfig.com/docs/functions/) | for everyone |
 | [language guide](docs/language-guide.md) | for everyone |
 | [function guide](docs/function-guide.md) | for mgmt developers |
 | [resource guide](docs/resource-guide.md) | for mgmt developers |
 | [style guide](docs/style-guide.md) | for mgmt developers |
+| [contributing guide](docs/contributing.md) | for mgmt contributors |
+| [service API guide](docs/service-guide.md) | for external developers |
 | [godoc API reference](https://godoc.org/github.com/purpleidea/mgmt) | for mgmt developers |
 | [prometheus guide](docs/prometheus.md) | for everyone |
+| [puppet guide](docs/puppet-guide.md) | for puppet sysadmins |
 | [development](docs/development.md) | for mgmt developers |
 | [videos](docs/on-the-web.md) | for everyone |
 | [blogs](docs/on-the-web.md) | for everyone |

cli/cli.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -119,6 +119,12 @@ type Args struct {
 
 	DeployCmd *DeployArgs `arg:"subcommand:deploy" help:"deploy code into a cluster"`
 
+	SetupCmd *SetupArgs `arg:"subcommand:setup" help:"setup some bootstrapping tasks"`
+
+	FirstbootCmd *FirstbootArgs `arg:"subcommand:firstboot" help:"run some tasks on first boot"`
+
+	DocsCmd *DocsGenerateArgs `arg:"subcommand:docs" help:"generate documentation"`
+
 	// This never runs, it gets preempted in the real main() function.
 	// XXX: Can we do it nicely with the new arg parser? can it ignore all args?
 	EtcdCmd *EtcdArgs `arg:"subcommand:etcd" help:"run standalone etcd"`
@@ -155,6 +161,18 @@ func (obj *Args) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
 		return cmd.Run(ctx, data)
 	}
 
+	if cmd := obj.SetupCmd; cmd != nil {
+		return cmd.Run(ctx, data)
+	}
+
+	if cmd := obj.FirstbootCmd; cmd != nil {
+		return cmd.Run(ctx, data)
+	}
+
+	if cmd := obj.DocsCmd; cmd != nil {
+		return cmd.Run(ctx, data)
+	}
+
 	// NOTE: we could return true, fmt.Errorf("...") if more than one did
 	return false, nil // nobody activated
 }

cli/deploy.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -61,6 +61,8 @@ type DeployArgs struct {
 	DeployEmpty      *cliUtil.EmptyArgs      `arg:"subcommand:empty" help:"deploy empty payload"`
 	DeployLang       *cliUtil.LangArgs       `arg:"subcommand:lang" help:"deploy lang (mcl) payload"`
 	DeployYaml       *cliUtil.YamlArgs       `arg:"subcommand:yaml" help:"deploy yaml graph payload"`
+	DeployPuppet     *cliUtil.PuppetArgs     `arg:"subcommand:puppet" help:"deploy puppet graph payload"`
+	DeployLangPuppet *cliUtil.LangPuppetArgs `arg:"subcommand:langpuppet" help:"deploy langpuppet graph payload"`
 }
 
 // Run executes the correct subcommand. It errors if there's ever an error. It
@@ -87,6 +89,14 @@ func (obj *DeployArgs) Run(ctx context.Context, data *cliUtil.Data) (bool, error
 		name = cliUtil.LookupSubcommand(obj, cmd) // "yaml"
 		args = cmd
 	}
+	if cmd := obj.DeployPuppet; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "puppet"
+		args = cmd
+	}
+	if cmd := obj.DeployLangPuppet; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "langpuppet"
+		args = cmd
+	}
 
 	// XXX: workaround https://github.com/alexflint/go-arg/issues/239
 	gapiNames := gapi.Names() // list of registered names

cli/docs.go

@@ -0,0 +1,150 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package cli
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	cliUtil "github.com/purpleidea/mgmt/cli/util"
+	"github.com/purpleidea/mgmt/docs"
+)
+
+// DocsGenerateArgs is the CLI parsing structure and type of the parsed result.
+// This particular one contains all the common flags for the `docs generate`
+// subcommand.
+type DocsGenerateArgs struct {
+	docs.Config // embedded config (can't be a pointer) https://github.com/alexflint/go-arg/issues/240
+
+	DocsGenerate *cliUtil.DocsGenerateArgs `arg:"subcommand:generate" help:"generate documentation"`
+}
+
+// Run executes the correct subcommand. It errors if there's ever an error. It
+// returns true if we did activate one of the subcommands. It returns false if
+// we did not. This information is used so that the top-level parser can return
+// usage or help information if no subcommand activates. This particular Run is
+// the run for the main `docs` subcommand.
+func (obj *DocsGenerateArgs) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var name string
+	var args interface{}
+	if cmd := obj.DocsGenerate; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "generate"
+		args = cmd
+	}
+	_ = name
+
+	Logf := func(format string, v ...interface{}) {
+		// Don't block this globally...
+		//if !data.Flags.Debug {
+		//	return
+		//}
+		data.Flags.Logf("main: "+format, v...)
+	}
+
+	var api docs.API
+
+	if cmd := obj.DocsGenerate; cmd != nil {
+		api = &docs.Generate{
+			DocsGenerateArgs: args.(*cliUtil.DocsGenerateArgs),
+			Config:           obj.Config,
+			Program:          data.Program,
+			Version:          data.Version,
+			Debug:            data.Flags.Debug,
+			Logf:             Logf,
+		}
+	}
+
+	if api == nil {
+		return false, nil // nothing found (display help!)
+	}
+
+	// We don't use these for the setup command in normal operation.
+	if data.Flags.Debug {
+		cliUtil.Hello(data.Program, data.Version, data.Flags) // say hello!
+		defer Logf("goodbye!")
+	}
+
+	// install the exit signal handler
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+	exit := make(chan struct{})
+	defer close(exit)
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+		// must have buffer for max number of signals
+		signals := make(chan os.Signal, 3+1) // 3 * ^C + 1 * SIGTERM
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+		var count uint8
+		for {
+			select {
+			case sig := <-signals: // any signal will do
+				if sig != os.Interrupt {
+					data.Flags.Logf("interrupted by signal")
+					return
+				}
+
+				switch count {
+				case 0:
+					data.Flags.Logf("interrupted by ^C")
+					cancel()
+				case 1:
+					data.Flags.Logf("interrupted by ^C (fast pause)")
+					cancel()
+				case 2:
+					data.Flags.Logf("interrupted by ^C (hard interrupt)")
+					cancel()
+				}
+				count++
+
+			case <-exit:
+				return
+			}
+		}
+	}()
+
+	if err := api.Main(ctx); err != nil {
+		if data.Flags.Debug {
+			data.Flags.Logf("main: %+v", err)
+		}
+		return false, err
+	}
+
+	return true, nil
+}

cli/firstboot.go

@@ -0,0 +1,151 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package cli
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	cliUtil "github.com/purpleidea/mgmt/cli/util"
+	"github.com/purpleidea/mgmt/firstboot"
+)
+
+// FirstbootArgs is the CLI parsing structure and type of the parsed result.
+// This particular one contains all the common flags for the `firstboot`
+// subcommand.
+type FirstbootArgs struct {
+	firstboot.Config // embedded config (can't be a pointer) https://github.com/alexflint/go-arg/issues/240
+
+	FirstbootStart *cliUtil.FirstbootStartArgs `arg:"subcommand:start" help:"start firstboot service"`
+}
+
+// Run executes the correct subcommand. It errors if there's ever an error. It
+// returns true if we did activate one of the subcommands. It returns false if
+// we did not. This information is used so that the top-level parser can return
+// usage or help information if no subcommand activates. This particular Run is
+// the run for the main `firstboot` subcommand. The firstboot command as a
+// service that lets you run commands once on the first boot of a system.
+func (obj *FirstbootArgs) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var name string
+	var args interface{}
+	if cmd := obj.FirstbootStart; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "pkg"
+		args = cmd
+	}
+	_ = name
+
+	Logf := func(format string, v ...interface{}) {
+		// Don't block this globally...
+		//if !data.Flags.Debug {
+		//	return
+		//}
+		data.Flags.Logf("main: "+format, v...)
+	}
+
+	var api firstboot.API
+
+	if cmd := obj.FirstbootStart; cmd != nil {
+		api = &firstboot.Start{
+			FirstbootStartArgs: args.(*cliUtil.FirstbootStartArgs),
+			Config:             obj.Config,
+			Program:            data.Program,
+			Version:            data.Version,
+			Debug:              data.Flags.Debug,
+			Logf:               Logf,
+		}
+	}
+
+	if api == nil {
+		return false, nil // nothing found (display help!)
+	}
+
+	// We don't use these for the setup command in normal operation.
+	if data.Flags.Debug {
+		cliUtil.Hello(data.Program, data.Version, data.Flags) // say hello!
+		defer Logf("goodbye!")
+	}
+
+	// install the exit signal handler
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+	exit := make(chan struct{})
+	defer close(exit)
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+		// must have buffer for max number of signals
+		signals := make(chan os.Signal, 3+1) // 3 * ^C + 1 * SIGTERM
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+		var count uint8
+		for {
+			select {
+			case sig := <-signals: // any signal will do
+				if sig != os.Interrupt {
+					data.Flags.Logf("interrupted by signal")
+					return
+				}
+
+				switch count {
+				case 0:
+					data.Flags.Logf("interrupted by ^C")
+					cancel()
+				case 1:
+					data.Flags.Logf("interrupted by ^C (fast pause)")
+					cancel()
+				case 2:
+					data.Flags.Logf("interrupted by ^C (hard interrupt)")
+					cancel()
+				}
+				count++
+
+			case <-exit:
+				return
+			}
+		}
+	}()
+
+	if err := api.Main(ctx); err != nil {
+		if data.Flags.Debug {
+			data.Flags.Logf("main: %+v", err)
+		}
+		return false, err
+	}
+
+	return true, nil
+}

cli/run.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -55,6 +55,8 @@ type RunArgs struct {
 	RunEmpty      *cliUtil.EmptyArgs      `arg:"subcommand:empty" help:"run empty payload"`
 	RunLang       *cliUtil.LangArgs       `arg:"subcommand:lang" help:"run lang (mcl) payload"`
 	RunYaml       *cliUtil.YamlArgs       `arg:"subcommand:yaml" help:"run yaml graph payload"`
+	RunPuppet     *cliUtil.PuppetArgs     `arg:"subcommand:puppet" help:"run puppet graph payload"`
+	RunLangPuppet *cliUtil.LangPuppetArgs `arg:"subcommand:langpuppet" help:"run a combined lang/puppet graph payload"`
 }
 
 // Run executes the correct subcommand. It errors if there's ever an error. It
@@ -81,6 +83,14 @@ func (obj *RunArgs) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
 		name = cliUtil.LookupSubcommand(obj, cmd) // "yaml"
 		args = cmd
 	}
+	if cmd := obj.RunPuppet; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "puppet"
+		args = cmd
+	}
+	if cmd := obj.RunLangPuppet; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "langpuppet"
+		args = cmd
+	}
 
 	// XXX: workaround https://github.com/alexflint/go-arg/issues/239
 	lists := [][]string{

cli/setup.go

@@ -0,0 +1,180 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package cli
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	cliUtil "github.com/purpleidea/mgmt/cli/util"
+	"github.com/purpleidea/mgmt/setup"
+)
+
+// SetupArgs is the CLI parsing structure and type of the parsed result. This
+// particular one contains all the common flags for the `setup` subcommand.
+type SetupArgs struct {
+	setup.Config // embedded config (can't be a pointer) https://github.com/alexflint/go-arg/issues/240
+
+	SetupPkg       *cliUtil.SetupPkgArgs       `arg:"subcommand:pkg" help:"setup packages"`
+	SetupSvc       *cliUtil.SetupSvcArgs       `arg:"subcommand:svc" help:"setup services"`
+	SetupFirstboot *cliUtil.SetupFirstbootArgs `arg:"subcommand:firstboot" help:"setup firstboot"`
+}
+
+// Run executes the correct subcommand. It errors if there's ever an error. It
+// returns true if we did activate one of the subcommands. It returns false if
+// we did not. This information is used so that the top-level parser can return
+// usage or help information if no subcommand activates. This particular Run is
+// the run for the main `setup` subcommand. The setup command does some
+// bootstrap work to help get things going.
+func (obj *SetupArgs) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var name string
+	var args interface{}
+	if cmd := obj.SetupPkg; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "pkg"
+		args = cmd
+	}
+	if cmd := obj.SetupSvc; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "svc"
+		args = cmd
+	}
+	if cmd := obj.SetupFirstboot; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "firstboot"
+		args = cmd
+	}
+	_ = name
+
+	Logf := func(format string, v ...interface{}) {
+		// Don't block this globally...
+		//if !data.Flags.Debug {
+		//	return
+		//}
+		data.Flags.Logf("main: "+format, v...)
+	}
+
+	var api setup.API
+
+	if cmd := obj.SetupPkg; cmd != nil {
+		api = &setup.Pkg{
+			SetupPkgArgs: args.(*cliUtil.SetupPkgArgs),
+			Config:       obj.Config,
+			Program:      data.Program,
+			Version:      data.Version,
+			Debug:        data.Flags.Debug,
+			Logf:         Logf,
+		}
+	}
+	if cmd := obj.SetupSvc; cmd != nil {
+		api = &setup.Svc{
+			SetupSvcArgs: args.(*cliUtil.SetupSvcArgs),
+			Config:       obj.Config,
+			Program:      data.Program,
+			Version:      data.Version,
+			Debug:        data.Flags.Debug,
+			Logf:         Logf,
+		}
+	}
+	if cmd := obj.SetupFirstboot; cmd != nil {
+		api = &setup.Firstboot{
+			SetupFirstbootArgs: args.(*cliUtil.SetupFirstbootArgs),
+			Config:             obj.Config,
+			Program:            data.Program,
+			Version:            data.Version,
+			Debug:              data.Flags.Debug,
+			Logf:               Logf,
+		}
+	}
+
+	if api == nil {
+		return false, nil // nothing found (display help!)
+	}
+
+	// We don't use these for the setup command in normal operation.
+	if data.Flags.Debug {
+		cliUtil.Hello(data.Program, data.Version, data.Flags) // say hello!
+		defer Logf("goodbye!")
+	}
+
+	// install the exit signal handler
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+	exit := make(chan struct{})
+	defer close(exit)
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+		// must have buffer for max number of signals
+		signals := make(chan os.Signal, 3+1) // 3 * ^C + 1 * SIGTERM
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+		var count uint8
+		for {
+			select {
+			case sig := <-signals: // any signal will do
+				if sig != os.Interrupt {
+					data.Flags.Logf("interrupted by signal")
+					return
+				}
+
+				switch count {
+				case 0:
+					data.Flags.Logf("interrupted by ^C")
+					cancel()
+				case 1:
+					data.Flags.Logf("interrupted by ^C (fast pause)")
+					cancel()
+				case 2:
+					data.Flags.Logf("interrupted by ^C (hard interrupt)")
+					cancel()
+				}
+				count++
+
+			case <-exit:
+				return
+			}
+		}
+	}()
+
+	if err := api.Main(ctx); err != nil {
+		if data.Flags.Debug {
+			data.Flags.Logf("main: %+v", err)
+		}
+		return false, err
+	}
+
+	return true, nil
+}

cli/util/args.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -102,3 +102,97 @@ type YamlArgs struct {
 	// Input is the input yaml code or file path or any input specification.
 	Input string `arg:"positional,required"`
 }
+
+// PuppetArgs is the puppet CLI parsing structure and type of the parsed result.
+type PuppetArgs struct {
+	// Input is the input puppet code or file path or just "agent".
+	Input string `arg:"positional,required"`
+
+	// PuppetConf is the optional path to a puppet.conf config file.
+	PuppetConf string `arg:"--puppet-conf" help:"full path to the puppet.conf file to use"`
+}
+
+// LangPuppetArgs is the langpuppet CLI parsing structure and type of the parsed
+// result.
+type LangPuppetArgs struct {
+	// LangInput is the input mcl code or file path or any input specification.
+	LangInput string `arg:"--lang,required" help:"the input parameter for the lang module"`
+
+	// PuppetInput is the input puppet code or file path or just "agent".
+	PuppetInput string `arg:"--puppet,required" help:"the input parameter for the puppet module"`
+
+	// copy-pasted from PuppetArgs
+
+	// PuppetConf is the optional path to a puppet.conf config file.
+	PuppetConf string `arg:"--puppet-conf" help:"full path to the puppet.conf file to use"`
+
+	// end PuppetArgs
+
+	// copy-pasted from LangArgs
+
+	// TODO: removed (temporarily?)
+	//Stdin bool `arg:"--stdin" help:"use passthrough stdin"`
+
+	Download     bool `arg:"--download" help:"download any missing imports"`
+	OnlyDownload bool `arg:"--only-download" help:"stop after downloading any missing imports"`
+	Update       bool `arg:"--update" help:"update all dependencies to the latest versions"`
+
+	OnlyUnify bool `arg:"--only-unify" help:"stop after type unification"`
+	SkipUnify bool `arg:"--skip-unify" help:"skip type unification"`
+
+	Depth int `arg:"--depth" default:"-1" help:"max recursion depth limit (-1 is unlimited)"`
+
+	// The default of 0 means any error is a failure by default.
+	Retry int `arg:"--depth" help:"max number of retries (-1 is unlimited)"`
+
+	ModulePath string `arg:"--module-path,env:MGMT_MODULE_PATH" help:"choose the modules path (absolute)"`
+
+	// end LangArgs
+}
+
+// SetupPkgArgs is the setup service CLI parsing structure and type of the
+// parsed result.
+type SetupPkgArgs struct {
+	Distro string `arg:"--distro" help:"build for this distro"`
+	Sudo   bool   `arg:"--sudo" help:"include sudo in the command"`
+	Exec   bool   `arg:"--exec" help:"actually run these commands"`
+}
+
+// SetupSvcArgs is the setup service CLI parsing structure and type of the
+// parsed result.
+type SetupSvcArgs struct {
+	BinaryPath string `arg:"--binary-path" help:"path to the binary"`
+	Install    bool   `arg:"--install" help:"install the systemd mgmt service"`
+	Start      bool   `arg:"--start" help:"start the mgmt service"`
+	Enable     bool   `arg:"--enable" help:"enable the mgmt service"`
+}
+
+// SetupFirstbootArgs is the setup service CLI parsing structure and type of the
+// parsed result.
+type SetupFirstbootArgs struct {
+	BinaryPath string `arg:"--binary-path" help:"path to the binary"`
+	Mkdir      bool   `arg:"--mkdir" help:"make the necessary firstboot dirs"`
+	Install    bool   `arg:"--install" help:"install the systemd firstboot service"`
+	Start      bool   `arg:"--start" help:"start the firstboot service (typically not used)"`
+	Enable     bool   `arg:"--enable" help:"enable the firstboot service"`
+
+	FirstbootStartArgs // Include these options if we want to specify them.
+}
+
+// FirstbootStartArgs is the firstboot service CLI parsing structure and type of
+// the parsed result.
+type FirstbootStartArgs struct {
+	LockFilePath string `arg:"--lock-file-path" help:"path to the lock file"`
+	ScriptsDir   string `arg:"--scripts-dir" help:"path to the scripts dir"`
+	DoneDir      string `arg:"--done-dir" help:"dir to move done scripts to"`
+	LoggingDir   string `arg:"--logging-dir" help:"directory to store logs in"`
+}
+
+// DocsGenerateArgs is the docgen utility CLI parsing structure and type of the
+// parsed result.
+type DocsGenerateArgs struct {
+	Output      string `arg:"--output" help:"output path to write to"`
+	RootDir     string `arg:"--root-dir" help:"path to mgmt source dir"`
+	NoResources bool   `arg:"--no-resources" help:"skip resource doc generation"`
+	NoFunctions bool   `arg:"--no-functions" help:"skip function doc generation"`
+}

cli/util/hello.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -41,7 +41,7 @@ func Hello(program, version string, flags Flags) {
 		program = "<unknown>"
 	}
 	fmt.Println(fmt.Sprintf("This is: %s, version: %s", program, version))
-	fmt.Println("Copyright (C) 2013-2024+ James Shubin and the project contributors")
+	fmt.Println("Copyright (C) James Shubin and the project contributors")
 	fmt.Println("Written by James Shubin <james@shubin.ca> and the project contributors")
 	flags.Logf("main: start: %v", start)
 }

cli/util/util.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

converger/converger.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

converger/converger_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

debian/copyright

@@ -3,7 +3,7 @@ Upstream-Name: mgmt
 Source: <https://github.com/purpleidea/mgmt>
 
 Files: *
-Copyright: Copyright (C) 2013-2024+ James Shubin and the project contributors
+Copyright: Copyright (C) James Shubin and the project contributors
 License: GPL-3.0
 
 License: GPL-3.0

doc.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20
+FROM golang:1.23
 
 MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
 

docker/Dockerfile-fedora.build

@@ -1,4 +1,4 @@
-FROM fedora:38
+FROM fedora:41
 LABEL org.opencontainers.image.authors="laurent.indermuehle@pm.me"
 
 ENV GOPATH=/root/gopath

docker/Dockerfile.build

@@ -6,7 +6,7 @@ ENV PATH=/opt/rh/rh-ruby22/root/usr/bin:/root/gopath/bin:/usr/local/sbin:/sbin:/
 ENV LD_LIBRARY_PATH=/opt/rh/rh-ruby22/root/usr/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
 ENV PKG_CONFIG_PATH=/opt/rh/rh-ruby22/root/usr/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}
 
-RUN yum -y install epel-release wget unzip git make which centos-release-scl gcc && sed -i "s/enabled=0/enabled=1/" /etc/yum.repos.d/epel-testing.repo && yum -y install rh-ruby22 && wget -O /opt/go1.20.11.linux-amd64.tar.gz https://storage.googleapis.com/golang/go1.20.11.linux-amd64.tar.gz && tar -C /usr/local -xzf /opt/go1.20.11.linux-amd64.tar.gz
+RUN yum -y install epel-release wget unzip git make which centos-release-scl gcc && sed -i "s/enabled=0/enabled=1/" /etc/yum.repos.d/epel-testing.repo && yum -y install rh-ruby22 && wget -O /opt/go1.23.5.linux-amd64.tar.gz https://storage.googleapis.com/golang/go1.23.5.linux-amd64.tar.gz && tar -C /usr/local -xzf /opt/go1.23.5.linux-amd64.tar.gz
 RUN mkdir -p $GOPATH/src/github.com/purpleidea && cd $GOPATH/src/github.com/purpleidea && git clone --recursive https://github.com/purpleidea/mgmt
 RUN go get -u gopkg.in/alecthomas/gometalinter.v1 && cd $GOPATH/src/github.com/purpleidea/mgmt && make deps && make build
 CMD ["/bin/bash"]

docker/Dockerfile.development

@@ -1,4 +1,4 @@
-FROM golang:1.20
+FROM golang:1.23
 
 MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
 

docs/conf.py

@@ -51,7 +51,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'mgmt'
-copyright = u'2013-2024+ James Shubin and the project contributors'
+copyright = u'Copyright (C) James Shubin and the project contributors'
 author = u'James Shubin'
 
 # The version info for the project you're documenting, acts as replacement for

docs/contributing.md

@@ -0,0 +1,96 @@
+# Contributing
+
+What follows is a short guide with information for participants who wish to
+contribute to the project. It hopes to set both some expectations and boundaries
+so that we both benefit.
+
+## Small patches
+
+If you have a small patch which you believe is straightforward, should be easy
+to merge, and isn't overly onerous on your time to write, please feel free to
+send it our way without asking first. Bug fixes are excellent examples of small
+patches. Please make sure to familiarize yourself with the rough coding style of
+the project first, and read through the [style guide](style-guide.md).
+
+## Making an excellent small patch
+
+As a special case: We'd like to avoid minimal effort, one-off, drive-by patches
+by bots and contributors looking to increase their "activity" numbers. As an
+example: a patch which fixes a small linting issue isn't rousing, but a patch
+that adds a linter test _and_ fixes a small linting issue is, because it shows
+you put in more effort.
+
+## Medium patches
+
+Medium sized patches are especially welcome. Good examples of these patches
+can include writing a new `mgmt` resource or function. You'll generally need
+some knowledge of golang interfaces and concurrency to write these patches.
+Before writing one of these, please make sure you understand some basics about
+the project and how the tool works. After this, it is recommended that you join
+our discussion channel to suggest the idea, and ideally include the actual API
+you'd like to propose before writing the code and sending a patch.
+
+## Making an excellent medium patch proposal
+
+The "API" of a resource is the type signature of the resource struct, and the
+"API" of a function is the type signature or signatures that it supports. (Since
+functions can be polymorphic, more than one signature can be possible!) A good
+proposal would likely also comment on the mechanisms the resources or functions
+would use to watch for events, to check state, and to apply changes. If these
+mechanisms need new dependencies, a brief survey of which dependencies are
+available and why you recommend a particular one is encouraged.
+
+## Large patches or structural and core patches
+
+Please do not send us large, core or structurally significant patches without
+first getting our approval and without getting some medium patches in first.
+These patches take a lot of effort to review, and we don't want to skimp on our
+commitment to that if we can't muster it. Instead grow our relationship with you
+on the medium-sized patches first. (A core patch might refer to something that
+touches either the function engine, resource engine, compiler internals, or
+something that is part of one of the internal API's.)
+
+## Expectations and boundaries
+
+When interacting with the project and soliciting feedback (either for design or
+during a code review) please keep in mind that the project (unfortunately!) has
+time constraints and so must prioritize how it handles workloads. If you are
+someone who has successfully sent in small patches, we will be more willing to
+spend time mentoring your medium sized patches and so on. Think of it this way:
+as you show that you're contributing to the project, we'll contribute more to
+you. Put another way: we can't afford to spend large amounts of time discussing
+potential patches with you, just to end up nowhere. Build up your reputation
+with us, and we hope to help grow our symbiosis with you all the while as you
+grow too!
+
+## Energy output
+
+The same goes for users and issue creators. There are times when we simply don't
+have the cycles to discuss or litigate an issue with you. We wish we did have
+more time, but it is finite, and running a project is not free. Therefore,
+please keep in mind that you don't automatically qualify for free support or
+attention.
+
+## Attention seeking behaviours
+
+Some folks spend too much time starting discussions, commenting on issues,
+"planning" and otherwise displaying attention seeking behaviours. Please avoid
+doing this as much as possible, especially if you are not already a major
+contributor to the project. While it may be well intentioned, if it is
+indistinguishable to us from intentional interference, then it's not welcome
+behaviour. Remember that Free Software is not free to write. If you require more
+attention, then either contribute more to the project, or consider paying for a
+[support contract](https://mgmtconfig.com/).
+
+## Consulting
+
+Having said all that, there are some folks who want to do some longer-term
+planning to decide if our core design and architecture is right for them to
+invest in. If that's the case, and you aren't already a well-known project
+contributor, please [contact](https://mgmtconfig.com/) us for a consulting
+quote. We have packages available for both individuals and businesses.
+
+## Respect
+
+Please be mindful and respectful of others when interacting with the project and
+its contributors. If you cannot abide by that, you may no longer be welcome.

docs/development.md

@@ -16,7 +16,7 @@ be working properly.
 
 ## Using Docker
 
-Alternatively, you can check out the [docker-guide](docker-guide.md) in order to
+Alternatively, you can check out the [docker folder](../docker/) in order to
 develop or deploy using docker. This method is not endorsed or supported, so use
 at your own risk, as it might not be working properly.
 
@@ -28,8 +28,9 @@ required for running the _test_ suite.
 
 ### Build
 
-* `golang` 1.20 or higher (required, available in some distros and distributed
-as a binary officially by [golang.org](https://golang.org/dl/))
+* A modern `golang` version. The version available in the current Fedora
+releases is usually supported. This is also distributed as a binary officially
+by [golang.org](https://golang.org/dl/).
 
 ### Runtime
 

docs/docs.go

@@ -0,0 +1,50 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+// Package docs provides a tool that generates documentation from the source.
+//
+// ./mgmt docs generate --output /tmp/docs.json && cat /tmp/docs.json | jq
+package docs
+
+import (
+	"context"
+)
+
+// API is the simple interface we expect for any setup items.
+type API interface {
+	// Main runs everything for this setup item.
+	Main(context.Context) error
+}
+
+// Config is a struct of all the configuration values which are shared by all of
+// the setup utilities. By including this as a separate struct, it can be used
+// as part of the API if we want.
+type Config struct {
+	//Foo string `arg:"--foo,env:MGMT_DOCGEN_FOO" help:"Foo..."` // TODO: foo
+}

docs/documentation.md

@@ -131,6 +131,33 @@ execute via a `remote` resource.
 You can read the introductory blog post about this topic here:
 [https://purpleidea.com/blog/2016/10/07/remote-execution-in-mgmt/](https://purpleidea.com/blog/2016/10/07/remote-execution-in-mgmt/)
 
+### Puppet support
+
+You can supply a puppet manifest instead of creating the (YAML) graph manually.
+Puppet must be installed and in `mgmt`'s search path. You also need the
+[ffrank-mgmtgraph puppet module](https://forge.puppet.com/ffrank/mgmtgraph).
+
+Invoke `mgmt` with the `--puppet` switch, which supports 3 variants:
+
+1. Request the configuration from the puppet server (like `puppet agent` does)
+
+	`mgmt run puppet --puppet agent`
+
+2. Compile a local manifest file (like `puppet apply`)
+
+	`mgmt run puppet --puppet /path/to/my/manifest.pp`
+
+3. Compile an ad hoc manifest from the commandline (like `puppet apply -e`)
+
+	`mgmt run puppet --puppet 'file { "/etc/ntp.conf": ensure => file }'`
+
+For more details and caveats see [puppet-guide.md](puppet-guide.md).
+
+#### Blog post
+
+An introductory post on the puppet support is on
+[Felix's blog](http://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/).
+
 ## Reference
 
 Please note that there are a number of undocumented options. For more
@@ -247,6 +274,29 @@ and it can't guarantee it if the resource is blocked because of a failed
 pre-requisite resource.
 *XXX: This is currently not implemented!*
 
+#### Dollar
+
+Boolean. Dollar allows you to have a resource name that starts with a `$` sign.
+This is false by default. This helps you catch cases when you write code like:
+
+```mcl
+$foo = "/tmp/file1"
+file "$foo" {} # incorrect!
+```
+
+The above code would ignore the `$foo` variable and attempt to make a file named
+`$foo` which would obviously not work. To correctly interpolate a variable, you
+need to surround the name with curly braces.
+
+```mcl
+$foo = "/tmp/file1"
+file "${foo}" {} # correct!
+```
+
+This meta param is a safety measure to make your life easier. It works for all
+resources. If someone comes up with a resource which would routinely start with
+a dollar sign, then we can revisit the default for this resource kind.
+
 #### Reverse
 
 Boolean. Reverse is a property that some resources can implement that specifies
@@ -335,7 +385,7 @@ size of 42, you can expect a semaphore if named: `:42`. It is expected that
 consumers of the semaphore metaparameter always include a prefix to avoid a
 collision with this globally defined semaphore. The size value must be greater
 than zero at this time. The traditional non-parallel execution found in config
-management tools such as `Puppet` can be obtained with `--sema 1`.
+management tools such as `puppet` can be obtained with `--sema 1`.
 
 #### `--ssh-priv-id-rsa`
 
@@ -410,7 +460,7 @@ directory in the git source repository. It is available from:
 
 ### Systemd:
 
-See [`misc/mgmt.service`](misc/mgmt.service) for a sample systemd unit file.
+See [`misc/mgmt.service`](../misc/mgmt.service) for a sample systemd unit file.
 This unit file is part of the RPM.
 
 To specify your custom options for `mgmt` on a systemd distro:
@@ -443,7 +493,7 @@ To report any bugs, please file a ticket at: [https://github.com/purpleidea/mgmt
 
 ## Authors
 
-Copyright (C) 2013-2024+ James Shubin and the project contributors
+Copyright (C) James Shubin and the project contributors
 
 Please see the
 [AUTHORS](https://github.com/purpleidea/mgmt/tree/master/AUTHORS) file

docs/faq.md

@@ -280,9 +280,76 @@ prevent masking an error for a situation when you expected a file to already be
 at that location. It also turns out to simplify the internals significantly, and
 remove an ambiguous scenario with the reversible file resource.
 
+### Package resources error with: "The name is not activatable", what's wrong?
+
+You may see an error like:
+
+`main: error running auto edges: The name is not activatable`
+
+This can happen because the mgmt `pkg` resource uses a library and daemon called
+`PackageKit` to install packages. If it is not installed, then it cannot do its
+work. On Fedora system you may wish to run `dnf install /usr/bin/pkcon` or on a
+Debian system you may wish to run `apt install packagekit-tools`.
+
+PackageKit is excellent because it provides both an API and an event system to
+watch the package database for changes, and it abstracts away the differences
+between the various package managers. If you'd prefer to not need to install
+this tool, then you can contribute a native `pkg:rpm` and `pkg:deb` resource to
+mgmt!
+
+### When running mgmt, it says: "module path error: can't find a module path".
+
+You might get an error along the lines of:
+
+```
+could not set scope: import scope `git://github.com/purpleidea/mgmt/modules/some_module_name/` failed: module path error: can't find a module path
+```
+
+This usually means that you haven't specified the directory that mgmt should use
+when looking for modules. This could happen when using mgmt interactively or
+when it's being run as a service. In such cases you may want the main invocation
+to look something like:
+
+```
+mgmt run lang --module-path '/etc/mgmt/modules/' /etc/mgmt/main.mcl
+```
+
+### I get an error: "cannot open shared object file: No such file or directory".
+
+Mgmt currently uses two libraries that depend on `.so` files being installed on
+the host. Those are for `augeas` and `libvirt`. If those dependencies are not
+present, then mgmt will not run. The complete error might look like:
+
+```
+mgmt: error while loading shared libraries: libvirt-lxc.so.0: cannot open shared object file: No such file or directory
+```
+
+or:
+
+```
+mgmt: error while loading shared libraries: libaugeas.so.0: cannot open shared object file: No such file or directory
+```
+
+or something similar. There are two solutions to this:
+
+1. Use a build that doesn't include one or both of those features. You can build
+that like: `GOTAGS="noaugeas novirt nodocker" make build`.
+
+2. Install those dependencies. On a Fedora machine you might want to run:
+
+```
+dnf install libvirt-devel augeas-devel
+```
+
+On a Debian machine you might want to run:
+
+```
+apt install libvirt-dev libaugeas-dev
+```
+
 ### Why do function names inside of templates include underscores?
 
-The golang template library which we use to implement the template() function
+The golang template library which we use to implement the golang.template() func
 doesn't support the dot notation, so we import all our normal functions, and
 just replace dots with underscores. As an example, the standard `datetime.print`
 function is shown within mcl scripts as datetime_print after being imported.
@@ -320,7 +387,7 @@ an instance of mgmt running, or if a related file locking issue occurred. To
 solve this, shutdown and running mgmt process, run `rm mgmt` to remove the file,
 and then get a new one by running `make` again.
 
-### Type unification error: "could not unify types: 2 unconsumed generators".
+### Type unification error with string interpolation.
 
 Look carefully at the following code:
 
@@ -343,8 +410,13 @@ print "hello" {
 }
 ```
 
-Yes we know the compiler gives horrible error messages, and yes we would
-absolutely love your help improving this.
+The first example will usually error with something along the lines of:
+
+`unify error with: topLevel(func() { <built-in:concat> }): type error: int != str`
+
+Now you know why this specific case doesn't work! We may reconsider allowing
+other types to be pulled into interpolation in the future. If you have a good
+case for this, then let us know.
 
 ### The run and deploy commands don't parse correctly when used with `--seeds`.
 

docs/function-guide.md

@@ -41,7 +41,7 @@ To implement a function, you'll need to create a file that imports the
 [`lang/funcs/simple/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/simple/)
 module. It should probably get created in the correct directory inside of:
 [`lang/core/`](https://github.com/purpleidea/mgmt/tree/master/lang/core/). The
-function should be implemented as a `FuncValue` in our type system. It is then
+function should be implemented as a `simple.Scaffold` in our API. It is then
 registered with the engine during `init()`. An example explains it best:
 
 ### Example
@@ -50,6 +50,7 @@ registered with the engine during `init()`. An example explains it best:
 package simple
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/purpleidea/mgmt/lang/funcs/simple"
@@ -59,9 +60,10 @@ import (
 // you must register your functions in init when the program starts up
 func init() {
 	// Example function that squares an int and prints out answer as an str.
-	simple.ModuleRegister(ModuleName, "talkingsquare", &types.FuncValue{
+
+	simple.ModuleRegister(ModuleName, "talkingsquare", &simple.Scaffold{
 		T: types.NewType("func(int) str"), // declare the signature
-		V: func(input []types.Value) (types.Value, error) {
+		F: func(ctx context.Context, input []types.Value) (types.Value, error) {
 			i := input[0].Int() // get first arg as an int64
 			// must return the above specified value
 			return &types.StrValue{
@@ -87,109 +89,41 @@ mgmt engine to shutdown. It should be seen as the equivalent to calling a
 Ideally, your functions should never need to error. You should never cause a
 real `panic()`, since this could have negative consequences to the system.
 
-## Simple Polymorphic Function API
-
-Most functions should be implemented using the simple function API. If they need
-to have multiple polymorphic forms under the same name, then you can use this
-API. This is useful for situations when it would be unhelpful to name the
-functions differently, or when the number of possible signatures for the
-function would be infinite.
-
-The canonical example of this is the `len` function which returns the number of
-elements in either a `list` or a `map`. Since lists and maps are two different
-types, you can see that polymorphism is more convenient than requiring a
-`listlen` and `maplen` function. Nevertheless, it is also required because a
-`list of int` is a different type than a `list of str`, which is a different
-type than a `list of list of str` and so on. As you can see the number of
-possible input types for such a `len` function is infinite.
-
-Another downside to implementing your functions with this API is that they will
-*not* be made available for use inside templates. This is a limitation of the
-`golang` template library. In the future if this limitation proves to be
-significantly annoying, we might consider writing our own template library.
-
-As with the simple, non-polymorphic API, you can only implement [pure](https://en.wikipedia.org/wiki/Pure_function)
-functions, without writing too much boilerplate code. They will be automatically
-re-evaluated as needed when their input values change.
-
-To implement a function, you'll need to create a file that imports the
-[`lang/funcs/simplepoly/`](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/simplepoly/)
-module. It should probably get created in the correct directory inside of:
-[`lang/core/`](https://github.com/purpleidea/mgmt/tree/master/lang/core/). The
-function should be implemented as a list of `FuncValue`'s in our type system. It
-is then registered with the engine during `init()`. You may also use the
-`variant` type in your type definitions. This special type will never be seen
-inside a running program, and will get converted to a concrete type if a
-suitable match to this signature can be found. Be warned that signatures which
-contain too many variants, or which are very general, might be hard for the
-compiler to match, and ambiguous type graphs make for user compiler errors. The
-top-level type must still be a function type, it may only contain variants as
-part of its signature. It is probably more difficult to unify a function if its
-return type is a variant, as opposed to if one of its args was.
-
-An example explains it best:
-
 ### Example
 
 ```golang
+package simple
+
 import (
+	"context"
 	"fmt"
 
-	"github.com/purpleidea/mgmt/lang/funcs/simplepoly"
+	"github.com/purpleidea/mgmt/lang/funcs/simple"
 	"github.com/purpleidea/mgmt/lang/types"
 )
 
 func init() {
-	// You may use the simplepoly.ModuleRegister method to register your
-	// function if it's in a module, as seen in the simple function example.
-	simplepoly.Register("len", []*types.FuncValue{
-		{
-			T: types.NewType("func([]variant) int"),
-			V: Len,
-		},
-		{
-			T: types.NewType("func({variant: variant}) int"),
-			V: Len,
-		},
+	// This is the actual definition of the `len` function.
+	simple.Register("len", &simple.Scaffold{
+		T: types.NewType("func(?1) int"), // contains a unification var
+		C: simple.TypeMatch([]string{     // match on any of these sigs
+			"func(str) int",
+			"func([]?1) int",
+			"func(map{?1: ?2}) int",
+		}),
+		// The implementation is left as an exercise for the reader.
+		F: Len,
 	})
 }
-
-// Len returns the number of elements in a list or the number of key pairs in a
-// map. It can operate on either of these types.
-func Len(input []types.Value) (types.Value, error) {
-	var length int
-	switch k := input[0].Type().Kind; k {
-	case types.KindList:
-		length = len(input[0].List())
-	case types.KindMap:
-		length = len(input[0].Map())
-
-	default:
-		return nil, fmt.Errorf("unsupported kind: %+v", k)
-	}
-
-	return &types.IntValue{
-		V: int64(length),
-	}, nil
-}
 ```
 
-This simple polymorphic function can accept an infinite number of signatures, of
-which there are two basic forms. Both forms return an `int` as is seen above.
-The first form takes a `[]variant` which means a `list` of `variant`'s, which
-means that it can be a list of any type, since `variant` itself is not a
-concrete type. The second form accepts a `{variant: variant}`, which means that
-it accepts any form of `map` as input.
+## Simple Polymorphic Function API
 
-The implementation for both of these forms is the same: it is handled by the
-same `Len` function which is clever enough to be able to deal with any of the
-type signatures possible from those two patterns.
-
-At compile time, if your `mcl` code type checks correctly, a concrete type will
-be known for each and every usage of the `len` function, and specific values
-will be passed in for this code to compute the length of. As usual, make sure to
-only write safe code that will not panic! A panic is a bug. If you really cannot
-continue, then you must return an error.
+Most functions should be implemented using the simple function API. If they need
+to have multiple polymorphic forms under the same name, with each resultant type
+match needing to be paired to a different implementation, then you can use this
+API. This is useful for situations when the functions differ in output type
+only.
 
 ## Function API
 
@@ -358,23 +292,6 @@ We don't expect this functionality to be particularly useful or common, as it's
 probably easier and preferable to simply import common golang library code into
 multiple different functions instead.
 
-## Polymorphic Function API
-
-The polymorphic function API is an API that lets you implement functions which
-do not necessarily have a single static function signature. After compile time,
-all functions must have a static function signature. We also know that there
-might be different ways you would want to call `printf`, such as:
-`printf("the %s is %d", "answer", 42)` or `printf("3 * 2 = %d", 3 * 2)`. Since
-you couldn't implement the infinite number of possible signatures, this API lets
-you write code which can be coerced into different forms. This makes
-implementing what would appear to be generic or polymorphic, instead of
-something that is actually static and that still has the static type safety
-properties that were guaranteed by the mgmt language.
-
-Since this is an advanced topic, it is not described in full at this time. For
-more information please have a look at the source code comments, some of the
-existing implementations, and ask around in the community.
-
 ## Frequently asked questions
 
 (Send your questions as a patch to this FAQ! I'll review it, merge it, and

docs/generate.go

@@ -0,0 +1,795 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package docs
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	cliUtil "github.com/purpleidea/mgmt/cli/util"
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
+	"github.com/purpleidea/mgmt/engine"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
+	"github.com/purpleidea/mgmt/lang/funcs"
+	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/util"
+)
+
+const (
+	// JSONSuffix is the output extension for the generated documentation.
+	JSONSuffix = ".json"
+)
+
+// Generate is the main entrypoint for this command. It generates everything.
+type Generate struct {
+	*cliUtil.DocsGenerateArgs // embedded config
+	Config                    // embedded Config
+
+	// Program is the name of this program, usually set at compile time.
+	Program string
+
+	// Version is the version of this program, usually set at compile time.
+	Version string
+
+	// Debug represents if we're running in debug mode or not.
+	Debug bool
+
+	// Logf is a logger which should be used.
+	Logf func(format string, v ...interface{})
+}
+
+// Main runs everything for this setup item.
+func (obj *Generate) Main(ctx context.Context) error {
+	if err := obj.Validate(); err != nil {
+		return err
+	}
+
+	if err := obj.Run(ctx); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Validate verifies that the structure has acceptable data stored within.
+func (obj *Generate) Validate() error {
+	if obj == nil {
+		return fmt.Errorf("data is nil")
+	}
+	if obj.Program == "" {
+		return fmt.Errorf("program is empty")
+	}
+	if obj.Version == "" {
+		return fmt.Errorf("version is empty")
+	}
+
+	return nil
+}
+
+// Run performs the desired actions to generate the documentation.
+func (obj *Generate) Run(ctx context.Context) error {
+
+	outputFile := obj.DocsGenerateArgs.Output
+	if outputFile == "" || !strings.HasSuffix(outputFile, JSONSuffix) {
+		return fmt.Errorf("must specify output")
+	}
+	// support relative paths too!
+	if !strings.HasPrefix(outputFile, "/") {
+		wd, err := os.Getwd()
+		if err != nil {
+			return err
+		}
+		outputFile = filepath.Join(wd, outputFile)
+	}
+
+	if obj.Debug {
+		obj.Logf("output: %s", outputFile)
+	}
+
+	// Ensure the directory exists.
+	//d := filepath.Dir(outputFile)
+	//if err := os.MkdirAll(d, 0750); err != nil {
+	//	return fmt.Errorf("could not make output dir at: %s", d)
+	//}
+
+	resources, err := obj.genResources()
+	if err != nil {
+		return err
+	}
+
+	functions, err := obj.genFunctions()
+	if err != nil {
+		return err
+	}
+
+	data := &Output{
+		Version:   safeVersion(obj.Version),
+		Resources: resources,
+		Functions: functions,
+	}
+
+	b, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+	b = append(b, '\n') // needs a trailing newline
+
+	if err := os.WriteFile(outputFile, b, 0600); err != nil {
+		return err
+	}
+	obj.Logf("wrote: %s", outputFile)
+
+	return nil
+}
+
+func (obj *Generate) getResourceInfo(kind, filename, structName string) (*ResourceInfo, error) {
+	rootDir := obj.DocsGenerateArgs.RootDir
+	if rootDir == "" {
+		wd, err := os.Getwd()
+		if err != nil {
+			return nil, err
+		}
+		rootDir = wd + "/" // add a trailing slash
+	}
+	if !strings.HasPrefix(rootDir, "/") || !strings.HasSuffix(rootDir, "/") {
+		return nil, fmt.Errorf("bad root dir: %s", rootDir)
+	}
+
+	// filename might be "noop.go" for example
+	p := filepath.Join(rootDir, engine.ResourcesRelDir, filename)
+
+	fset := token.NewFileSet()
+
+	// f is a: https://golang.org/pkg/go/ast/#File
+	f, err := parser.ParseFile(fset, p, nil, parser.ParseComments)
+	if err != nil {
+		return nil, err
+	}
+
+	// mcl field name to golang field name
+	mapping, err := engineUtil.LangFieldNameToStructFieldName(kind)
+	if err != nil {
+		return nil, err
+	}
+	// golang field name to mcl field name
+	nameMap, err := util.MapSwap(mapping)
+	if err != nil {
+		return nil, err
+	}
+	// mcl field name to mcl type
+	typMap, err := engineUtil.LangFieldNameToStructType(kind)
+	if err != nil {
+		return nil, err
+	}
+
+	ri := &ResourceInfo{}
+	// Populate the fields, even if they don't have a comment.
+	ri.Name = structName // golang name
+	ri.Kind = kind       // duplicate data
+	ri.File = filename
+	ri.Fields = make(map[string]*ResourceFieldInfo)
+	for mclFieldName, fieldName := range mapping {
+		typ, exists := typMap[mclFieldName]
+		if !exists {
+			continue
+		}
+
+		ri.Fields[mclFieldName] = &ResourceFieldInfo{
+			Name: fieldName,
+			Type: typ.String(),
+			Desc: "", // empty for now
+		}
+	}
+
+	var previousComment *ast.CommentGroup
+
+	// Walk through the AST...
+	ast.Inspect(f, func(node ast.Node) bool {
+
+		// Comments above the struct appear as a node right _before_ we
+		// find the struct, so if we see one, save it for later...
+		if cg, ok := node.(*ast.CommentGroup); ok {
+			previousComment = cg
+			return true
+		}
+
+		typeSpec, ok := node.(*ast.TypeSpec)
+		if !ok {
+			return true
+		}
+		name := typeSpec.Name.Name // name is now known!
+
+		// If the struct isn't what we're expecting, then move on...
+		if name != structName {
+			return true
+		}
+
+		// Check if the TypeSpec is a named struct type that we want...
+		st, ok := typeSpec.Type.(*ast.StructType)
+		if !ok {
+			return true
+		}
+
+		// At this point, we have the struct we want...
+
+		var comment *ast.CommentGroup
+		if typeSpec.Doc != nil {
+			// I don't know how to even get here...
+			comment = typeSpec.Doc // found!
+
+		} else if previousComment != nil {
+			comment = previousComment // found!
+			previousComment = nil
+		}
+
+		ri.Desc = commentCleaner(comment)
+
+		// Iterate over the fields of the struct
+		for _, field := range st.Fields.List {
+			// Check if the field has a comment associated with it
+			if field.Doc == nil {
+				continue
+			}
+
+			if len(field.Names) < 1 { // XXX: why does this happen?
+				continue
+			}
+
+			fieldName := field.Names[0].Name
+			if fieldName == "" { // Can this happen?
+				continue
+			}
+			if isPrivate(fieldName) {
+				continue
+			}
+
+			mclFieldName, exists := nameMap[fieldName]
+			if !exists {
+				continue
+			}
+
+			ri.Fields[mclFieldName].Desc = commentCleaner(field.Doc)
+		}
+
+		return true
+	})
+
+	return ri, nil
+}
+
+func (obj *Generate) genResources() (map[string]*ResourceInfo, error) {
+	resources := make(map[string]*ResourceInfo)
+	if obj.DocsGenerateArgs.NoResources {
+		return resources, nil
+	}
+
+	r := engine.RegisteredResourcesNames()
+	sort.Strings(r)
+	for _, kind := range r {
+		metadata, err := docsUtil.LookupResource(kind)
+		if err != nil {
+			return nil, err
+		}
+
+		if strings.HasPrefix(kind, "_") {
+			// TODO: Should we display these somehow?
+			// built-in resource
+			continue
+		}
+
+		ri, err := obj.getResourceInfo(kind, metadata.Filename, metadata.Typename)
+		if err != nil {
+			return nil, err
+		}
+
+		if ri.Name == "" {
+			return nil, fmt.Errorf("empty resource name: %s", kind)
+		}
+		if ri.File == "" {
+			return nil, fmt.Errorf("empty resource file: %s", kind)
+		}
+		if ri.Desc == "" {
+			obj.Logf("empty resource desc: %s", kind)
+		}
+		fields := []string{}
+		for field := range ri.Fields {
+			fields = append(fields, field)
+		}
+		sort.Strings(fields)
+		for _, field := range fields {
+			if ri.Fields[field].Desc == "" {
+				obj.Logf("empty resource (%s) field desc: %s", kind, field)
+			}
+		}
+
+		resources[kind] = ri
+	}
+
+	return resources, nil
+}
+
+func (obj *Generate) getFunctionInfo(pkg, name string, metadata *docsUtil.Metadata) (*FunctionInfo, error) {
+	rootDir := obj.DocsGenerateArgs.RootDir
+	if rootDir == "" {
+		wd, err := os.Getwd()
+		if err != nil {
+			return nil, err
+		}
+		rootDir = wd + "/" // add a trailing slash
+	}
+	if !strings.HasPrefix(rootDir, "/") || !strings.HasSuffix(rootDir, "/") {
+		return nil, fmt.Errorf("bad root dir: %s", rootDir)
+	}
+	if metadata.Filename == "" {
+		return nil, fmt.Errorf("empty filename for: %s.%s", pkg, name)
+	}
+
+	// filename might be "pow.go" for example and contain a rel dir
+	p := filepath.Join(rootDir, funcs.FunctionsRelDir, metadata.Filename)
+
+	fset := token.NewFileSet()
+
+	// f is a: https://golang.org/pkg/go/ast/#File
+	f, err := parser.ParseFile(fset, p, nil, parser.ParseComments)
+	if err != nil {
+		return nil, err
+	}
+
+	fi := &FunctionInfo{}
+	fi.Name = metadata.Typename
+	fi.File = metadata.Filename
+
+	var previousComment *ast.CommentGroup
+	found := false
+
+	rawFunc := func(node ast.Node) (*ast.CommentGroup, string) {
+		fd, ok := node.(*ast.FuncDecl)
+		if !ok {
+			return nil, ""
+		}
+		return fd.Doc, fd.Name.Name // name is now known!
+	}
+
+	rawStruct := func(node ast.Node) (*ast.CommentGroup, string) {
+		typeSpec, ok := node.(*ast.TypeSpec)
+		if !ok {
+			return nil, ""
+		}
+
+		// Check if the TypeSpec is a named struct type that we want...
+		if _, ok := typeSpec.Type.(*ast.StructType); !ok {
+			return nil, ""
+		}
+
+		return typeSpec.Doc, typeSpec.Name.Name // name is now known!
+	}
+
+	// Walk through the AST...
+	ast.Inspect(f, func(node ast.Node) bool {
+
+		// Comments above the struct appear as a node right _before_ we
+		// find the struct, so if we see one, save it for later...
+		if cg, ok := node.(*ast.CommentGroup); ok {
+			previousComment = cg
+			return true
+		}
+
+		doc, name := rawFunc(node) // First see if it's a raw func.
+		if name == "" {
+			doc, name = rawStruct(node) // Otherwise it's a struct.
+		}
+
+		// If the func isn't what we're expecting, then move on...
+		if name != metadata.Typename {
+			return true
+		}
+
+		var comment *ast.CommentGroup
+		if doc != nil {
+			// I don't know how to even get here...
+			comment = doc // found!
+
+		} else if previousComment != nil {
+			comment = previousComment // found!
+			previousComment = nil
+		}
+
+		fi.Desc = commentCleaner(comment)
+		found = true
+
+		return true
+	})
+
+	if !found {
+		//return nil, nil
+	}
+
+	return fi, nil
+}
+
+func (obj *Generate) genFunctions() (map[string]*FunctionInfo, error) {
+	functions := make(map[string]*FunctionInfo)
+	if obj.DocsGenerateArgs.NoFunctions {
+		return functions, nil
+	}
+
+	m := funcs.Map() // map[string]func() interfaces.Func
+	names := []string{}
+	for name := range m {
+		names = append(names, name)
+	}
+	sort.Slice(names, func(i, j int) bool {
+		a := names[i]
+		b := names[j]
+		// TODO: do a sorted-by-package order.
+		return a < b
+	})
+
+	for _, name := range names {
+		//v := m[name]
+		//fn := v()
+		fn := m[name]()
+
+		// eg: golang/strings.has_suffix
+		sp := strings.Split(name, ".")
+		if len(sp) == 0 {
+			return nil, fmt.Errorf("unexpected empty function")
+		}
+		if len(sp) > 2 {
+			return nil, fmt.Errorf("unexpected function name: %s", name)
+		}
+		n := sp[0]
+		p := sp[0]
+		if len(sp) == 1 { // built-in
+			p = "" // no package!
+		}
+		if len(sp) == 2 { // normal import
+			n = sp[1]
+		}
+
+		if strings.HasPrefix(n, "_") {
+			// TODO: Should we display these somehow?
+			// built-in function
+			continue
+		}
+
+		var sig *string
+		//iface := ""
+		if x := fn.Info().Sig; x != nil {
+			s := x.String()
+			sig = &s
+			//iface = "simple"
+		}
+
+		metadata := &docsUtil.Metadata{}
+
+		// XXX: maybe we need a better way to get this?
+		mdFunc, ok := fn.(interfaces.MetadataFunc)
+		if !ok {
+			// Function doesn't tell us what the data is, let's try
+			// to get it automatically...
+			metadata.Typename = funcs.GetFunctionName(fn) // works!
+			metadata.Filename = ""                        // XXX: How can we get this?
+
+			// XXX: We only need this back-channel metadata store
+			// because we don't know how to get the filename without
+			// manually writing code in each function. Alternatively
+			// we could add a New() method to each struct and then
+			// we could modify the struct instead of having it be
+			// behind a copy which is needed to get new copies!
+			var err error
+			metadata, err = docsUtil.LookupFunction(name)
+			if err != nil {
+				return nil, err
+			}
+
+		} else if mdFunc == nil {
+			// programming error
+			return nil, fmt.Errorf("unexpected empty metadata for function: %s", name)
+
+		} else {
+			metadata = mdFunc.GetMetadata()
+		}
+
+		if metadata == nil {
+			return nil, fmt.Errorf("unexpected nil metadata for function: %s", name)
+		}
+
+		// This may be an empty func name if the function did not know
+		// how to get it. (This is normal for automatic regular funcs.)
+		if metadata.Typename == "" {
+			metadata.Typename = funcs.GetFunctionName(fn) // works!
+		}
+
+		fi, err := obj.getFunctionInfo(p, n, metadata)
+		if err != nil {
+			return nil, err
+		}
+		// We may not get any fields added if we can't find anything...
+		fi.Name = metadata.Typename
+		fi.Package = p
+		fi.Func = n
+		fi.File = metadata.Filename
+		//fi.Desc = desc
+		fi.Signature = sig
+
+		// Hack for golang generated functions!
+		if strings.HasPrefix(fi.Package, "golang/") && fi.File == "generated_funcs.go" {
+			pkg := fi.Package[len("golang/"):]
+			frag := strings.TrimPrefix(fi.Name, strings.Title(strings.Join(strings.Split(pkg, "/"), ""))) // yuck
+			fi.File = fmt.Sprintf("https://pkg.go.dev/%s#%s", pkg, frag)
+		}
+
+		if fi.Func == "" {
+			return nil, fmt.Errorf("empty function name: %s", name)
+		}
+		if fi.File == "" {
+			return nil, fmt.Errorf("empty function file: %s", name)
+		}
+		if fi.Desc == "" {
+			obj.Logf("empty function desc: %s", name)
+		}
+		if fi.Signature == nil {
+			obj.Logf("empty function sig: %s", name)
+		}
+
+		functions[name] = fi
+	}
+
+	return functions, nil
+}
+
+// Output is the type of the final data that will be for the json output.
+type Output struct {
+	// Version is the sha1 or ref name of this specific version. This is
+	// used if we want to generate documentation with links matching the
+	// correct version. If unspecified then this assumes git master.
+	Version string `json:"version"`
+
+	// Resources contains the collection of every available resource!
+	// FIXME: should this be a list instead?
+	Resources map[string]*ResourceInfo `json:"resources"`
+
+	// Functions contains the collection of every available function!
+	// FIXME: should this be a list instead?
+	Functions map[string]*FunctionInfo `json:"functions"`
+}
+
+// ResourceInfo stores some information about each resource.
+type ResourceInfo struct {
+	// Name is the golang name of this resource.
+	Name string `json:"name"`
+
+	// Kind is the kind of this resource.
+	Kind string `json:"kind"`
+
+	// File is the file name where this resource exists.
+	File string `json:"file"`
+
+	// Desc explains what this resource does.
+	Desc string `json:"description"`
+
+	// Fields is a collection of each resource field and corresponding info.
+	Fields map[string]*ResourceFieldInfo `json:"fields"`
+}
+
+// ResourceFieldInfo stores some information about each field in each resource.
+type ResourceFieldInfo struct {
+	// Name is what this field is called in golang format.
+	Name string `json:"name"`
+
+	// Type is the mcl type for this field.
+	Type string `json:"type"`
+
+	// Desc explains what this field does.
+	Desc string `json:"description"`
+}
+
+// FunctionInfo stores some information about each function.
+type FunctionInfo struct {
+	// Name is the golang name of this function. This may be an actual
+	// function if used by the simple API, or the name of a struct.
+	Name string `json:"name"`
+
+	// Package is the import name to use to get to this function.
+	Package string `json:"package"`
+
+	// Func is the name of the function in that package.
+	Func string `json:"func"`
+
+	// File is the file name where this function exists.
+	File string `json:"file"`
+
+	// Desc explains what this function does.
+	Desc string `json:"description"`
+
+	// Signature is the type signature of this function. If empty then the
+	// signature is not known statically and it may be polymorphic.
+	Signature *string `json:"signature,omitempty"`
+}
+
+// commentCleaner takes a comment group and returns it as a clean string. It
+// removes the spurious newlines and programmer-focused comments. If there are
+// blank lines, it replaces them with a single newline. The idea is that the
+// webpage formatter would replace the newline with a <br /> or similar. This
+// code is a modified alternative of the ast.CommentGroup.Text() function.
+func commentCleaner(g *ast.CommentGroup) string {
+	if g == nil {
+		return ""
+	}
+	comments := make([]string, len(g.List))
+	for i, c := range g.List {
+		comments[i] = c.Text
+	}
+
+	lines := make([]string, 0, 10) // most comments are less than 10 lines
+	for _, c := range comments {
+		// Remove comment markers.
+		// The parser has given us exactly the comment text.
+		switch c[1] {
+		case '/':
+			//-style comment (no newline at the end)
+			c = c[2:]
+			if len(c) == 0 {
+				// empty line
+				break
+			}
+			if isDevComment(c[1:]) { // get rid of one space
+				continue
+			}
+			if c[0] == ' ' {
+				// strip first space - required for Example tests
+				c = c[1:]
+				break
+			}
+			//if isDirective(c) {
+			//	// Ignore //go:noinline, //line, and so on.
+			//	continue
+			//}
+		case '*':
+			/*-style comment */
+			c = c[2 : len(c)-2]
+		}
+
+		// Split on newlines.
+		cl := strings.Split(c, "\n")
+
+		// Walk lines, stripping trailing white space and adding to list.
+		for _, l := range cl {
+			lines = append(lines, stripTrailingWhitespace(l))
+		}
+	}
+
+	// Remove leading blank lines; convert runs of interior blank lines to a
+	// single blank line.
+	n := 0
+	for _, line := range lines {
+		if line != "" || n > 0 && lines[n-1] != "" {
+			lines[n] = line
+			n++
+		}
+	}
+	lines = lines[0:n]
+
+	// Concatenate all of these together. Blank lines should be a newline.
+	s := ""
+	for i, line := range lines {
+		if line == "" {
+			continue
+		}
+		s += line
+		if i < len(lines)-1 { // Is there another line?
+			if lines[i+1] == "" {
+				s += "\n" // Will eventually be a line break.
+			} else {
+				s += " "
+			}
+		}
+	}
+
+	return s
+}
+
+// TODO: should we use unicode.IsSpace instead?
+func isWhitespace(ch byte) bool { return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' }
+
+// TODO: should we replace with a strings package stdlib function?
+func stripTrailingWhitespace(s string) string {
+	i := len(s)
+	for i > 0 && isWhitespace(s[i-1]) {
+		i--
+	}
+	return s[0:i]
+}
+
+// isPrivate specifies if a field name is "private" or not.
+func isPrivate(fieldName string) bool {
+	if fieldName == "" {
+		panic("invalid field name")
+	}
+	x := fieldName[0:1]
+
+	if strings.ToLower(x) == x {
+		return true // it was already private
+	}
+
+	return false
+}
+
+// isDevComment tells us that the comment is for developers only!
+func isDevComment(comment string) bool {
+	if strings.HasPrefix(comment, "TODO:") {
+		return true
+	}
+	if strings.HasPrefix(comment, "FIXME:") {
+		return true
+	}
+	if strings.HasPrefix(comment, "XXX:") {
+		return true
+	}
+	return false
+}
+
+// safeVersion parses the main version string and returns a short hash for us.
+// For example, we might get a string of 0.0.26-176-gabcdef012-dirty as input,
+// and we'd want to return abcdef012.
+func safeVersion(version string) string {
+	const dirty = "-dirty"
+
+	s := version
+	if strings.HasSuffix(s, dirty) { // helpful dirty remover
+		s = s[0 : len(s)-len(dirty)]
+	}
+
+	ix := strings.LastIndex(s, "-")
+	if ix == -1 { // assume we have a standalone version (future proofing?)
+		return s
+	}
+	s = s[ix+1:]
+
+	// From the `git describe` man page: The "g" prefix stands for "git" and
+	// is used to allow describing the version of a software depending on
+	// the SCM the software is managed with. This is useful in an
+	// environment where people may use different SCMs.
+	const g = "g"
+	if strings.HasPrefix(s, g) {
+		s = s[len(g):]
+	}
+
+	return s
+}

docs/index.rst

@@ -14,3 +14,4 @@ Welcome to mgmt's documentation!
    quick-start-guide
    resource-guide
    prometheus
+   puppet-guide

docs/language-guide.md

@@ -283,6 +283,14 @@ one of many ways you can perform iterative tasks that you might have
 traditionally used a `for` loop for instead. This is preferred, because flow
 control is error-prone and can make for less readable code.
 
+The single `str` variation, may only be used when it is possible for the
+compiler to determine statically that the value is of that type. Otherwise, it
+will assume it to be a list of strings. Programmers should explicitly wrap their
+variables in a string by interpolation to force this static `str` determination,
+or in square brackets to force a list. The former is generally preferable
+because it generates a smaller function graph since it doesn't need to build a
+list.
+
 ##### Internal edges
 
 Resources may also declare edges internally. The edges may point to or from
@@ -337,6 +345,28 @@ to express a relationship between three resources. The first character in the
 resource kind must be capitalized so that the parser can't ascertain
 unambiguously that we are referring to a dependency relationship.
 
+##### Edge naming
+
+Each edge must have a unique name of type `str` that is used to uniquely
+identify that edge, and can be used in the functioning of the edge at its
+discretion.
+
+Alternatively, the name value may be a list of strings `[]str` to build a list
+of edges, each with a name from that list.
+
+Using this construct is a veiled form of looping (iteration). This technique is
+one of many ways you can perform iterative tasks that you might have
+traditionally used a `for` loop for instead. This is preferred, because flow
+control is error-prone and can make for less readable code.
+
+The single `str` variation, may only be used when it is possible for the
+compiler to determine statically that the value is of that type. Otherwise, it
+will assume it to be a list of strings. Programmers should explicitly wrap their
+variables in a string by interpolation to force this static `str` determination,
+or in square brackets to force a list. The former is generally preferable
+because it generates a smaller function graph since it doesn't need to build a
+list.
+
 #### Class
 
 A class is a grouping structure that bind's a list of statements to a name in
@@ -561,7 +591,7 @@ Lexing is done using [nex](https://github.com/blynn/nex). It is a pure-golang
 implementation which is similar to _Lex_ or _Flex_, but which produces golang
 code instead of C. It integrates reasonably well with golang's _yacc_ which is
 used for parsing. The token definitions are in:
-[lang/lexer.nex](https://github.com/purpleidea/mgmt/tree/master/lang/lexer.nex).
+[lang/lexer.nex](https://github.com/purpleidea/mgmt/tree/master/lang/parser/lexer.nex).
 Lexing and parsing run together by calling the `LexParse` method.
 
 #### Parsing
@@ -573,7 +603,7 @@ and trial and error. One small advantage yacc has over standard yacc is that it
 can produce error messages from examples. The best documentation is to examine
 the source. There is a short write up available [here](https://research.swtch.com/yyerror).
 The yacc file exists at:
-[lang/parser.y](https://github.com/purpleidea/mgmt/tree/master/lang/parser.y).
+[lang/parser.y](https://github.com/purpleidea/mgmt/tree/master/lang/parser/parser.y).
 Lexing and parsing run together by calling the `LexParse` method.
 
 #### Interpolation
@@ -609,23 +639,27 @@ so that each `Expr` node in the AST knows what to expect. Type annotation is
 allowed in situations when you want to explicitly specify a type, or when the
 compiler cannot deduce it, however, most of it can usually be inferred.
 
-For type inferrence to work, each node in the AST implements a `Unify` method
-which is able to return a list of invariants that must hold true. This starts at
-the top most AST node, and gets called through to it's children to assemble a
-giant list of invariants. The invariants can take different forms. They can
-specify that a particular expression must have a particular type, or they can
-specify that two expressions must have the same types. More complex invariants
-allow you to specify relationships between different types and expressions.
-Furthermore, invariants can allow you to specify that only one invariant out of
-a set must hold true.
+For type inference to work, each `Stmt` node in the AST implements a `TypeCheck`
+method which is able to return a list of invariants that must hold true. This
+starts at the top most AST node, and gets called through to it's children to
+assemble a giant list of invariants. The invariants all have the same form. They
+specify that a particular expression corresponds to two particular types which
+may both contain unification variables.
+
+Each `Expr` node in the AST implements an `Infer` and `Check` method. The
+`Infer` method returns the type of that node along with a list of invariants as
+described above. Unification variables can of course be used throughout. The
+`Check` method always uses a generic check implementation and generally doesn't
+need to be implemented by the user.
 
 Once the list of invariants has been collected, they are run through an
 invariant solver. The solver can return either return successfully or with an
-error. If the solver returns successfully, it means that it has found a trivial
+error. If the solver returns successfully, it means that it has found a single
 mapping between every expression and it's corresponding type. At this point it
 is a simple task to run `SetType` on every expression so that the types are
-known. If the solver returns in error, it is usually due to one of two
-possibilities:
+known. During this stage, each SetType method verifies that it's a compatible
+type that it can use. If either that method or if the solver returns in error,
+it is usually due to one of two possibilities:
 
 1. Ambiguity
 
@@ -645,8 +679,8 @@ possibilities:
 	always happens if the user has made a type error in their program.
 
 Only one solver currently exists, but it is possible to easily plug in an
-alternate implementation if someone more skilled in the art of solver design
-would like to propose a more logical or performant variant.
+alternate implementation if someone wants to experiment with the art of solver
+design and would like to propose a more logical or performant variant.
 
 #### Function graph generation
 
@@ -687,8 +721,9 @@ If you'd like to create a built-in, core function, you'll need to implement the
 function API interface named `Func`. It can be found in
 [lang/interfaces/func.go](https://github.com/purpleidea/mgmt/tree/master/lang/interfaces/func.go).
 Your function must have a specific type. For example, a simple math function
-might have a signature of `func(x int, y int) int`. As you can see, all the
-types are known _before_ compile time.
+might have a signature of `func(x int, y int) int`. The simple functions have
+their types known _before_ compile time. You may also include unification
+variables in the function signature as long as the top-level type is a function.
 
 A separate discussion on this matter can be found in the [function guide](function-guide.md).
 
@@ -716,6 +751,12 @@ added in the future. This method is usually called before any other, and should
 not depend on any other method being called first. Other methods must not depend
 on this method being called first.
 
+If you use any unification variables in the function signature, then your
+function will *not* be made available for use inside templates. This is a
+limitation of the `golang` templating library. In the future if this limitation
+proves to be significantly annoying, we might consider writing our own template
+library.
+
 #### Example
 
 ```golang
@@ -726,6 +767,18 @@ func (obj *FooFunc) Info() *interfaces.Info {
 }
 ```
 
+#### Example
+
+This example contains unification variables.
+
+```golang
+func (obj *FooFunc) Info() *interfaces.Info {
+	return &interfaces.Info{
+		Sig: types.NewType("func(a ?1, b ?2, foo [?3]) ?1"),
+	}
+}
+```
+
 ### Init
 
 ```golang
@@ -788,49 +841,67 @@ Please see the example functions in
 [lang/core/](https://github.com/purpleidea/mgmt/tree/master/lang/core/).
 ```
 
-### Polymorphic Function API
+### BuildableFunc Function API
 
-For some functions, it might be helpful to be able to implement a function once,
-but to have multiple polymorphic variants that can be chosen at compile time.
-For this more advanced topic, you will need to use the
-[Polymorphic Function API](#polymorphic-function-api). This will help with code
-reuse when you have a small, finite number of possible type signatures, and also
-for more complicated cases where you might have an infinite number of possible
-type signatures. (eg: `[]str`, or `[][]str`, or `[][][]str`, etc...)
+For some functions, it might be helpful to have a function which needs a "build"
+step which is run after type unification. This step can be used to build the
+function using the determined type, but it may also just be used for checking
+that unification picked a valid solution.
 
 Suppose you want to implement a function which can assume different type
 signatures. The mgmt language does not support polymorphic types-- you must use
 static types throughout the language, however, it is legal to implement a
 function which can take different specific type signatures based on how it is
 used. For example, you might wish to add a math function which could take the
-form of `func(x int, x int) int` or `func(x float, x float) float` depending on
-the input values. You might also want to implement a function which takes an
-arbitrary number of input arguments (the number must be statically fixed at the
-compile time of your program though) and which returns a string.
+form of `func(x int, y int) int` or `func(x float, y float) float` depending on
+the input values. For this case you could use a signature containing unification
+variables, eg: `func(x ?1, y ?1) ?1`. At the end the buildable function would
+need to check that it received a `?1` type of either `int` or `float`, since
+this function might not support doing math on strings. Remember that type
+unification can only return zero or one solutions, it's not possible to return
+more than one, which is why this secondary validation step is a brilliant way to
+filter out invalid solutions without needing to encode them as algebraic
+conditions during the solver state, which would otherwise make it exponential.
 
-The `PolyFunc` interface adds additional methods which you must implement to
-satisfy such a function implementation. If you'd like to implement such a
-function, then please notify the project authors, and they will expand this
-section with a longer description of the process.
+### InferableFunc Function API
 
-#### Examples
+You might also want to implement a function which takes an arbitrary number of
+input arguments (the number must be statically fixed at the compile time of your
+program though) and which returns a string or something else.
 
-What follows are a few examples that might help you understand some of the
-language details.
+The `InferableFunc` interface adds ad additional `FuncInfer` method which you
+must implement to satisfy such a function implementation. This lets you
+dynamically generate a type signature (including unification variables) and a
+list of invariants before running the type unification solver. It takes as input
+a list of the statically known input types and input values (if any) and as well
+the number of input arguments specified. This is usually enough information to
+generate a fixed type signature of a fixed size.
 
-##### Example Foo
-
-TODO: please add an example here!
-
-##### Example Bar
-
-TODO: please add an example here!
+Using this API should generally be pretty rare, but it is how certain special
+functions such as `fmt.printf` are built. If you'd like to implement such a
+function, then please notify the project authors as we're curious about your
+use case.
 
 ## Frequently asked questions
 
 (Send your questions as a patch to this FAQ! I'll review it, merge it, and
 respond by commit with the answer.)
 
+### Why am I getting a deploy.readfile error when the file actually exists?
+
+You may be seeing an error like:
+
+`readfile`: open /*/files/foo: file does not exist can't read file `/files/foo`?
+
+If you look, the `foo` file is indeed in the `files/` directory. The problem is
+that the `files/` directory won't be seen if you didn't specify to include it as
+part of your deploy. To do so, chances are that all you need to do is add a
+`metadata.yaml` file into the parent directory to that files folder. This will
+be used as the entrypoint instead of the naked `main.mcl` file that you have
+there, and with that metadata entrypoint, you get a default `files/` directory
+added. You can of course change the `files/` path by setting a key in the
+`metadata.yaml` file, but we recommend you leave it as the default.
+
 ### What is the difference between `ExprIf` and `StmtIf`?
 
 The language contains both an `if` expression, and and `if` statement. An `if`

docs/on-the-web.md

@@ -21,15 +21,15 @@ if we missed something that you think is relevant!
 | Felix Frank | blog | [Puppet Powered Mgmt (puppet to mgmt tl;dr)](https://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/) |
 | James Shubin | blog | [Automatic clustering in mgmt](https://purpleidea.com/blog/2016/06/20/automatic-clustering-in-mgmt/) |
 | James Shubin | video | [Recording from CoreOSFest 2016](https://www.youtube.com/watch?v=KVmDCUA42wc&html5=1) |
-| James Shubin | video | [Recording from DebConf16](http://meetings-archive.debian.net/pub/debian-meetings/2016/debconf16/Next_Generation_Config_Mgmt.webm) ([Slides](https://annex.debconf.org//debconf-share/debconf16/slides/15-next-generation-config-mgmt.pdf)) |
+| James Shubin | video | [Recording from DebConf16](http://meetings-archive.debian.net/pub/debian-meetings/2016/debconf16/Next_Generation_Config_Mgmt.webm) |
 | Felix Frank | blog | [Edging It All In (puppet and mgmt edges)](https://ffrank.github.io/features/2016/07/12/edging-it-all-in/) |
 | Felix Frank | blog | [Translating All The Things (puppet to mgmt translation warnings)](https://ffrank.github.io/features/2016/08/19/translating-all-the-things/) |
-| James Shubin | video | [Recording from systemd.conf 2016](https://www.youtube.com/watch?v=jB992Zb3nH0&html5=1) |
+| James Shubin | video | [Recording from systemd.conf 2016](https://www.youtube.com/watch?v=_TowsFAWWRA) |
 | James Shubin | blog | [Remote execution in mgmt](https://purpleidea.com/blog/2016/10/07/remote-execution-in-mgmt/) |
-| James Shubin | video | [Recording from High Load Strategy 2016](https://vimeo.com/191493409) |
-| James Shubin | video | [Recording from NLUUG 2016](https://www.youtube.com/watch?v=MmpwOQAb_SE&html5=1) |
+| James Shubin | video | [Recording from High Load Strategy 2016](https://www.youtube.com/watch?v=-4g14KUVPVk) |
+| James Shubin | video | [Recording from NLUUG 2016](https://www.youtube.com/watch?v=0vO93ni1zos) |
 | James Shubin | blog | [Send/Recv in mgmt](https://purpleidea.com/blog/2016/12/07/sendrecv-in-mgmt/) |
-| Julien Pivotto | blog | [Augeas resource for mgmt](https://roidelapluie.be/blog/2017/02/14/mgmt-augeas/) |
+| Julien Pivotto | blog | [Augeas resource for mgmt](https://purpleidea.com/cached/mgmt-augeas.html) (Cached from: https://roidelapluie.be/blog/2017/02/14/mgmt-augeas/) |
 | James Shubin | blog | [Metaparameters in mgmt](https://purpleidea.com/blog/2017/03/01/metaparameters-in-mgmt/) |
 | James Shubin | video | [Recording from Incontro DevOps 2017](https://vimeo.com/212241877) |
 | Yves Brissaud | blog | [mgmt aux HumanTalks Grenoble (french)](http://log.winsos.net/2017/04/12/mgmt-aux-human-talks-grenoble.html) |
@@ -59,3 +59,5 @@ if we missed something that you think is relevant!
 | James Shubin | video | [Recording from CfgMgmtCamp.eu 2023](https://www.youtube.com/watch?v=FeRGRj8w0BU) |
 | James Shubin | video | [Recording from FOSDEM 2024, Golang Devroom](https://video.fosdem.org/2024/ud2218a/fosdem-2024-2575-single-binary-full-stack-provisioning.mp4) |
 | James Shubin | video | [Recording from CfgMgmtCamp.eu 2024](https://www.youtube.com/watch?v=vBt9lpGD4bc) |
+| James Shubin | blog | [Mgmt Configuration Language: Functions](https://purpleidea.com/blog/2024/11/22/functions-in-mgmt/) |
+| James Shubin | blog | [Modules and imports in mgmt](https://purpleidea.com/blog/2024/12/03/modules-and-imports-in-mgmt/) |

docs/puppet-guide.md

@@ -0,0 +1,316 @@
+# Puppet guide
+
+`mgmt` can use Puppet as its source for the configuration graph.
+This document goes into detail on how this works, and lists
+some pitfalls and limitations.
+
+For basic instructions on how to use the Puppet support, see
+the [main documentation](documentation.md#puppet-support).
+
+## Prerequisites
+
+You need Puppet installed in your system. It is not important how you
+get it. On the most common Linux distributions, you can use packages
+from the OS maintainer, or upstream Puppet repositories. An alternative
+that will also work on OSX is the `puppet` Ruby gem. It also has the
+advantage that you can install any desired version in your home directory
+or any other location.
+
+Any release of Puppet's 3.x and 4.x series should be suitable for use with
+`mgmt`. Most importantly, make sure to install the `ffrank-mgmtgraph` Puppet
+module (referred to below as "the translator module").
+
+```
+puppet module install ffrank-mgmtgraph
+```
+
+Please note that the module is not required on your Puppet master (if you
+use a master/agent setup). It's needed on the machine that runs `mgmt`.
+You can install the module on the master anyway, so that it gets distributed
+to your agents through Puppet's `pluginsync` mechanism.
+
+### Testing the Puppet side
+
+The following command should run successfully and print a YAML hash on your
+terminal:
+
+```puppet
+puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": ensure => present }'
+```
+
+You can use this CLI to test any manifests before handing them straight
+to `mgmt`.
+
+## Writing a suitable manifest
+
+### Unsupported attributes
+
+`mgmt` inherited its resource module from Puppet, so by and large, it's quite
+possible to express `mgmt` graphs in terms of Puppet manifests. However,
+there isn't (and likely never will be) full feature parity between the
+respective resource types. In consequence, a manifest can have semantics that
+cannot be transferred to `mgmt`.
+
+For example, at the time of writing this, the `file` type in `mgmt` had no
+notion of permissions (the file `mode`) yet. This lead to the following
+warning (among others that will be discussed below):
+
+```
+$ puppet mgmtgraph print --code 'file { "/tmp/foo": mode => "0600" }'
+Warning: cannot translate: File[/tmp/foo] { mode => "600" } (attribute is ignored)
+```
+
+This is a heads-up for the user, because the resulting `mgmt` graph will
+in fact not pass this information to the `/tmp/foo` file resource, and
+`mgmt` will ignore this file's permissions. Including such attributes in
+manifests that are written expressly for `mgmt` is not sensible and should
+be avoided.
+
+### Unsupported resources
+
+Puppet has a fairly large number of
+[built-in types](https://www.puppet.com/docs/puppet/8/cheatsheet_core_types.html),
+and countless more are available through
+[modules](https://forge.puppet.com/). It's unlikely that all of them will
+eventually receive native counterparts in `mgmt`.
+
+When encountering an unknown resource, the translator module will replace
+it with an `exec` resource in its output. This resource will run the equivalent
+of a `puppet resource` command to make Puppet apply the original resource
+itself. This has quite abysmal performance, because processing such a
+resource requires the forking of at least one Puppet process (two if it
+is found to be out of sync). This comes with considerable overhead.
+On most systems, starting up any Puppet command takes several seconds.
+Compared to the split second that the actual work usually takes,
+this overhead can amount to several orders of magnitude.
+
+Avoid Puppet types that `mgmt` does not implement (yet).
+
+### Avoiding common warnings
+
+Many resource parameters in Puppet take default values. For the most part,
+the translator module just ignores them. However, there are cases in which
+Puppet will default to convenient behavior that `mgmt` cannot quite replicate.
+For example, translating a plain `file` resource will lead to a warning message:
+
+```
+$ puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": }'
+Warning: File[/tmp/mgmt-test] uses the 'puppet' file bucket, which mgmt cannot
+do. There will be no backup copies!
+```
+
+The reason is that per default, Puppet assumes the following parameter value
+(among others)
+
+```puppet
+file { "/tmp/mgmt-test":
+	backup => 'puppet',
+}
+```
+
+To avoid this, specify the parameter explicitly:
+
+```bash
+puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": backup => false }'
+```
+
+This is tedious in a more complex manifest. A good simplification is the
+following [resource default](https://www.puppet.com/docs/puppet/8/lang_defaults)
+anywhere on the top scope of your manifest:
+
+```puppet
+File { backup => false }
+```
+
+If you encounter similar warnings from other types and/or parameters,
+use the same approach to silence them if possible.
+
+## Configuring Puppet
+
+Since `mgmt` uses an actual Puppet CLI behind the scenes, you might
+need to tweak some of Puppet's runtime options in order to make it
+do what you want. Reasons for this could be among the following:
+
+* You use the `--puppet agent` variant and need to configure
+`servername`, `certname` and other master/agent-related options.
+* You don't want runtime information to end up in the `vardir`
+that is used by your regular `puppet agent`.
+* You install specific Puppet modules for `mgmt` in a non-standard
+location.
+
+`mgmt` exposes only one Puppet option in order to allow you to
+control all of them, through its `--puppet-conf` option. It allows
+you to specify which `puppet.conf` file should be used during
+translation.
+
+```
+mgmt run puppet --puppet /opt/my-manifest.pp --puppet-conf /etc/mgmt/puppet.conf
+```
+
+Within this file, you can just specify any needed options in the
+`[main]` section:
+
+```
+[main]
+server=mgmt-master.example.net
+vardir=/var/lib/mgmt/puppet
+```
+
+## Caveats
+
+Please see the [README](https://github.com/ffrank/puppet-mgmtgraph/blob/master/README.md)
+of the translator module for the current state of supported and unsupported
+language features.
+
+You should probably make sure to always use the latest release of
+both `ffrank-mgmtgraph` and `ffrank-yamlresource` (the latter is
+getting pulled in as a dependency of the former).
+
+## Using Puppet in conjunction with the mcl lang
+
+The graph that Puppet generates for `mgmt` can be united with a graph
+that is created from native `mgmt` code in its mcl language. This is
+useful when you are in the process of replacing Puppet with mgmt. You
+can translate your custom modules into mgmt's language one by one,
+and let mgmt run the current mix.
+
+Instead of the usual `--puppet-conf` flag and argv for `puppet` and `mcl` input,
+you need to use alternative flags to make this work:
+
+* `--lp-lang` to specify the mcl input
+* `--lp-puppet` to specify the puppet input
+* `--lp-puppet-conf` to point to the optional puppet.conf file
+
+`mgmt` will derive a graph that contains all edges and vertices from
+both inputs. You essentially get two unrelated subgraphs that run in
+parallel. To form edges between these subgraphs, you have to define
+special vertices that will be merged. This works through a hard-coded
+naming scheme.
+
+### Mixed graph example 1 - No merges
+
+```mcl
+# lang
+file "/tmp/mgmt_dir/" { state => "present" }
+file "/tmp/mgmt_dir/a" { state => "present" }
+```
+
+```puppet
+# puppet
+file { "/tmp/puppet_dir": ensure => "directory" }
+file { "/tmp/puppet_dir/a": ensure => "file" }
+```
+
+These very simple inputs (including implicit edges from directory to
+respective file) result in two subgraphs that do not relate.
+
+```
+File[/tmp/mgmt_dir/] -> File[/tmp/mgmt_dir/a]
+
+File[/tmp/puppet_dir] -> File[/tmp/puppet_dir/a]
+```
+
+### Mixed graph example 2 - Merged vertex
+
+In order to have merged vertices in the resulting graph, you will
+need to include special resources and classes in the respective
+input code.
+
+* On the lang side, add `noop` resources with names starting in `puppet_`.
+* On the Puppet side, add **empty** classes with names starting in `mgmt_`.
+
+```mcl
+# lang
+noop "puppet_handover_to_mgmt" {}
+file "/tmp/mgmt_dir/" { state => "present" }
+file "/tmp/mgmt_dir/a" { state => "present" }
+
+Noop["puppet_handover_to_mgmt"] -> File["/tmp/mgmt_dir/"]
+```
+
+```puppet
+# puppet
+class mgmt_handover_to_mgmt {}
+include mgmt_handover_to_mgmt
+
+file { "/tmp/puppet_dir": ensure => "directory" }
+file { "/tmp/puppet_dir/a": ensure => "file" }
+
+File["/tmp/puppet_dir/a"] -> Class["mgmt_handover_to_mgmt"]
+```
+
+The new `noop` resource is merged with the new class, resulting in
+the following graph:
+
+```
+File[/tmp/puppet_dir] -> File[/tmp/puppet_dir/a]
+				|
+				V
+		Noop[handover_to_mgmt]
+			|
+			V
+	File[/tmp/mgmt_dir/] -> File[/tmp/mgmt_dir/a]
+```
+
+You put all your ducks in a row, and the resources from the Puppet input
+run before those from the mcl input.
+
+**Note:** The names of the `noop` and the class must be identical after the
+respective prefix. The common part (here, `handover_to_mgmt`) becomes the name
+of the merged resource.
+
+## Mixed graph example 3 - Multiple merges
+
+In most scenarios, it will not be possible to define a single handover
+point like in the previous example. For example, if some Puppet resources
+need to run in between two stages of native resources, you need at least
+two merged vertices:
+
+```mcl
+# lang
+noop "puppet_handover" {}
+noop "puppet_handback" {}
+file "/tmp/mgmt_dir/" { state => "present" }
+file "/tmp/mgmt_dir/a" { state => "present" }
+file "/tmp/mgmt_dir/puppet_subtree/state-file" { state => "present" }
+
+File["/tmp/mgmt_dir/"] -> Noop["puppet_handover"]
+Noop["puppet_handback"] -> File["/tmp/mgmt_dir/puppet_subtree/state-file"]
+```
+
+```puppet
+# puppet
+class mgmt_handover {}
+class mgmt_handback {}
+
+include mgmt_handover, mgmt_handback
+
+class important_stuff {
+	file { "/tmp/mgmt_dir/puppet_subtree":
+		ensure => "directory"
+	}
+	# ...
+}
+
+Class["mgmt_handover"] -> Class["important_stuff"] -> Class["mgmt_handback"]
+```
+
+The resulting graph looks roughly like this:
+
+```
+File[/tmp/mgmt_dir/] -> File[/tmp/mgmt_dir/a]
+	|
+	V
+Noop[handover] -> ( class important_stuff resources )
+			|
+			V
+		Noop[handback]
+			|
+			V
+File[/tmp/mgmt_dir/puppet_subtree/state-file]
+```
+
+You can add arbitrary numbers of merge pairs to your code bases,
+with relationships as needed. From our limited experience, code
+readability suffers quite a lot from these, however. We advise
+to keep these structures simple.

docs/quick-start-guide.md

@@ -21,8 +21,6 @@ to build your own.
 
 ### Downloading a pre-built release:
 
-This method is not recommended because those packages are now very old.
-
 The latest releases can be found [here](https://github.com/purpleidea/mgmt/releases/).
 An alternate mirror is available [here](https://dl.fedoraproject.org/pub/alt/purpleidea/mgmt/releases/).
 
@@ -39,7 +37,7 @@ You'll need some dependencies, including `golang`, and some associated tools.
 
 #### Installing golang
 
-* You need golang version 1.20 or greater installed.
+* You need a modern golang version installed.
 	* To install on rpm style systems: `sudo dnf install golang`
 	* To install on apt style systems: `sudo apt install golang`
 	* To install on macOS systems install [Homebrew](https://brew.sh)
@@ -103,13 +101,14 @@ This method avoids polluting your workstation with the dependencies for the
 build. Here is an example using Fedora, Podman and Buildah:
 
 ```shell
-git clone --recursive https://github.com/purpleidea/mgmt/ ~/mgmt/
-cd ~/mgmt/docker
-buildah build -f Dockerfile-fedora.build -t mgmt_build
-podman run -d -it --name mgmt_build localhost/mgmt_build
-podman cp mgmt_build:/src/github.com/purpleidea/mgmt/mgmt /tmp/mgmt
-sudo mv /tmp/mgmt /usr/local/bin  # be sure this is in your $PATH
-sudo chown root:root /usr/local/bin/mgmt
+git clone --recursive https://github.com/purpleidea/mgmt/
+cd mgmt
+docker build -t mgmt -f docker/Dockerfile .
+docker run --rm --entrypoint cat mgmt mgmt > mgmt
+chmod +x mgmt
+./mgmt --version
+# you could now copy the mgmt binary somewhere into your $PATH
+# e.g., /usr/local/bin/ to make it accessible from anywhere
 ```
 
 ## Running mgmt

docs/release-notes/0.0.26

@@ -0,0 +1,142 @@
+I've just released version 0.0.26 of mgmt!
+
+> 16 files changed, 869 insertions(+), 181 deletions(-)
+
+Hot off the heels of the recent large release (0.0.25) I've just
+released an incremental update...
+
+See more here:
+
+https://purpleidea.com/blog/2024/03/27/a-new-provisioning-tool/
+
+With that, here are a few highlights from the release:
+
+* We have a new mgmt partner program. Please sign-up for early access
+to these release notes, along with other special privileges. Details
+at: https://bit.ly/mgmt-partner-program
+
+* Type unification for the provisioning tool is about 40x faster.
+
+* We fix a small bug related to the upcoming fedora 40 release.
+
+And much more...
+
+
+DOWNLOAD
+
+Prebuilt binaries are available here for this release:
+https://github.com/purpleidea/mgmt/releases/tag/0.0.26
+
+They can also be found on the Fedora mirror:
+https://dl.fedoraproject.org/pub/alt/purpleidea/mgmt/releases/0.0.26/
+
+
+NEWS
+
+* Added old release notes into git
+
+* We now skip over unreleased Fedora versions (like "40 Beta") when
+trying to automatically determine the latest stable release.
+
+* Type unification was structurally refactored to make way for a bunch
+of future improvements and generally to modernize the code.
+
+* Added some unification optimizations and a unification flag
+optimizations system to allow solvers to support special flags. One of
+these new flags was used for the provisioner code with a substantial
+improvement in type unification time by about 40x.
+
+* New cli args are also available for using these flags.
+
+* We're looking for help writing Amazon, Google, DigitalOcean, Hetzner,
+etc, resources if anyone is interested, reach out to us. Particularly
+if there is support from those organizations as well.
+
+* Many other bug fixes, changes, etc...
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+
+BUGS/TODO
+
+* Function values getting _passed_ to resources doesn't work yet, but
+it's not a blocker, but it would definitely be useful. We're looking
+into it.
+
+* Function graphs are unnecessarily dynamic. We might make them more
+static so that we don't need as many transactions. This is really a
+compiler optimization and not a bug, but it's something important we'd
+like to have.
+
+* Running two Txn's during the same pause would be really helpful. I'm
+not sure how much of a performance improvement we'd get from this, but
+it would sure be interesting to build. If you want to build a fancy
+synchronization primitive, then let us know! Again this is not a bug.
+
+* General type unification performance can be improved drastically. I
+will have to implement the fast algorithm so that we can scale to very
+large mcl programs. Help is wanted if you are familiar with "unionfind"
+and/or type unification.
+
+
+TALKS
+
+I don't have anything planned until CfgMgmtCamp 2025. If you'd like to
+book me for a private event, or sponsor my travel for your conference,
+please let me know.
+
+I recently gave two talks: one at CfgMgmtCamp 2024, and one at FOSDEM
+in the golang room. Both are available online and demonstrated an
+earlier version of the provisioning tool which is fully available
+today. The talks can be found here: https://purpleidea.com/talks/
+
+
+PARTNER PROGRAM
+
+We have a new mgmt partner program which gets you early access to
+releases, bug fixes, support, and many other goodies. Please sign-up
+today: https://bit.ly/mgmt-partner-program
+
+
+MISC
+
+Our mailing list host (Red Hat) is no longer letting non-Red Hat
+employees use their infrastructure. We're looking for a new home. I've
+opened a ticket with Freedesktop. If you have any sway with them or
+other recommendations, please let me know:
+https://gitlab.freedesktop.org/freedesktop/freedesktop/-/issues/1082
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Libera IRC or Matrix (preferred) and
+ping us if you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many tagged #mgmtlove issues exist:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+Although asking in IRC/matrix is the best way to find something to work
+on.
+
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is fun and friendly! You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+James Shubin
+We had 1 unique committers since 0.0.25, and have had 90 overall.
+
+
+Happy hacking,
+James
+@purpleidea

docs/resource-guide.md

@@ -60,7 +60,10 @@ it is not specified, but others cannot, and some might poorly infer if the
 struct name is ambiguous.
 
 If you'd like your resource to be accessible by the `YAML` graph API (GAPI),
-then you'll need to include the appropriate YAML fields as shown below.
+then you'll need to include the appropriate YAML fields as shown below. This is
+used by the `puppet` compiler as well, so make sure you include these struct
+tags if you want existing `puppet` code to be able to run using the `mgmt`
+engine.
 
 #### Example
 
@@ -620,7 +623,7 @@ func init() { // special golang method that runs once
 
 To support YAML unmarshalling for your resource, you must implement an
 additional method. It is recommended if you want to use your resource with the
-`yaml` compiler.
+`puppet` compiler.
 
 ```golang
 UnmarshalYAML(unmarshal func(interface{}) error) error // optional
@@ -713,7 +716,7 @@ Higher level resource collections will be possible once the `mgmt` DSL is ready.
 ### Why does the resource API have `CheckApply` instead of two separate methods?
 
 In an early version we actually had both "parts" as separate methods, namely:
-`StateOK` (Check) and `Apply`, but the [decision](58f41eddd9c06b183f889f15d7c97af81b0331cc)
+`StateOK` (Check) and `Apply`, but the [decision](https://github.com/purpleidea/mgmt/commit/58f41eddd9c06b183f889f15d7c97af81b0331cc)
 was made to merge the two into a single method. There are two reasons for this:
 
 1. Many situations would involve the engine running both `Check` and `Apply`. If

docs/service-guide.md

@@ -0,0 +1,145 @@
+# Service API design guide
+
+This document is intended as a short instructional design guide in building a
+service management API. It is certainly intended for someone who wishes to use
+`mgmt` resources and functions to interact with their facilities, however it may
+be of more general use as well. Hopefully this will help you make smarter design
+considerations early on, and prevent some amount of unnecessary technical debt.
+
+## Main aspects
+
+What follows are some of the most common considerations which you may wish to
+take into account when building your service. This list is non-exhaustive. Of
+particular note, as of the writing of this document, many of these designs are
+not taken into account or not well-handled or implemented by the major API
+("cloud") providers.
+
+### Authentication
+
+#### The status-quo
+
+Many services naturally require you to authenticate yourself. Usually the
+initial user who sets up the account and provides credit card details will need
+to download secret credentials in order to access the service. The onus is on
+the user to keep those credentials private, and to prevent leaking them. It is
+convenient (and insecure) to store them in `git` repositories containing scripts
+and configuration management code. Since it's likely you will use multiple
+different services, it also means you will have a ton of different credentials
+to guard.
+
+#### An alternative
+
+Instead, build your service to accept a public key that you store in the users
+account. Only consumers that can correctly sign messages matching this public
+key should be authorized. This mechanism is well-understood by anyone who has
+ever uploaded their public SSH key to a server. You can use SSH keys, GPG keys,
+or even get into Kerberos if that's appropriate. Best of all, if you and other
+services use a standardized mechanism like GPG, a user might only need to keep
+track of their single key-pair, even when they're using multiple services!
+
+### Events
+
+#### The problem
+
+People have been building "[CRUD](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete)"
+and "[REST](https://en.wikipedia.org/wiki/REST)"ful API's for years. The biggest
+missing part that most of them don't provide is events. If users want to know
+when a resource changes, they have to repeatedly poll the server, which is both
+network intensive, and introduces latency. When services were simpler, this
+wasn't as much of a consideration, but these days it matters. An embarrassingly
+small number of major software vendors implement these correctly, if at all.
+
+#### Why events?
+
+The `mgmt` tool is different from most other static tools in that it allows
+reading streams of incoming data, and stream of change events from resources we
+are managing. If an event API is not available, we can still poll, but this is
+not as desirable. An event-capable API doesn't prevent polling if that's
+preferred, you can always repeat a read request periodically.
+
+#### Variants
+
+The two common mechanisms for receiving events are "callbacks" and
+"long-polling". In the former, the service contacts the consumer when something
+happens. In the latter, the consumer opens a connection, and the service either
+closes the connection or sends the reply, when it's ready. Long-polling is often
+preferred since it doesn't require an open firewall on the consumers side.
+Callbacks are preferred because it's often cheaper for the service to implement
+that. It's also less reliable since it's hard to know if the callback message
+wasn't received because it was dropped, or if there just wasn't an event. And it
+requires static timeouts when retrying a callback message, and so on. It's best
+to implement long-polling or something equivalent at a minimum.
+
+#### "Since" requests
+
+When making an event request, some API's will let you tack on a "since" style
+parameter that tells the endpoint that we're interested in all of the events
+_since_ a particular timestamp, or _since_ a particular sequence ID. This can be
+very useful if missing an intermediate event is a concern. Implement this if you
+can, but it's better for all concerned if purely declarative facilities are all
+that is required. It also forces the endpoint to maintain some state, which may
+be undesirable for them.
+
+#### Out of band
+
+Some providers have the event system tacked on to a separate facility. If it's
+not part of the core API, then it's not useful. You shouldn't have to configure
+a separate system in order to start getting events.
+
+### Batching
+
+With so many resources, you might expect to have 1000's of long-polling
+connections all sitting open and idle. That can't be efficient! It's not, which
+is why good API's need a batching facility. This lets the consumer group
+together many watches (all waiting on a long-poll) inside of a single call. That
+way, a single connection might only be needed for a large amount of information.
+
+### Don't auto-generate junk
+
+Please build an elegant API. Many services auto-generate a "phone book" SDK of
+junk. It might seem inevitable, so if you absolutely need to do this, then put
+some extra effort into making it idiomatic. If I'm using an SDK generated for
+`golang` and I see an internal `foo.String` wrapper, then chances are you have
+designed your API and code to be easier to maintain for you, instead of
+prioritizing your customers. Surely the total volume of all customer code is
+more than your own, so why optimize for that instead of the putting the customer
+first?
+
+### Resources and functions
+
+`Mgmt` has a concept of "resources" and "functions". Resources are used in an
+idempotent model to express desired state and perform that work, and "functions"
+are used to receive and pull data into the system. That separation has shown to
+be an elegant one. Consider it when designing your API's. For example, if some
+vital information can only be obtained after performing a modifying operation,
+then it might signal that you're missing some sort of a lookup or event-log
+system. Design your API's to be idempotent, this solves many distributed-system
+problems involving receiving duplicate messages, and so on.
+
+## Using mgmt as a library
+
+Instead of building a new service from scratch, and re-inventing the typical
+management and CLI layer, consider using `mgmt` as a library, and directly
+benefiting from that work. This has not been done for a large production
+service, but the author believes it would be quite efficient, particularly if
+your application is written in golang. It's equivalently easy to do it for other
+languages as well, you just end up with two binaries instead of one. (Or you can
+embed the other binary into the new golang management tool.)
+
+## Cloud API considerations
+
+Many "cloud" companies have a lot of technical debt and a lot of customers. As a
+result, it might be very hard for them to improve their API's, particularly
+without breaking compatibility promises for their existing customers. As a
+result, they should either add a versioned API, which lets newer consumers get
+the benefit, or add new parallel services which offer the modern features. If
+they don't, the only solution is for new competitors to build-in these better
+efficiencies, eventually offering better value to cost ratios, which will then
+make legacy products less lucrative and therefore unmaintainable as compared to
+their competitors.
+
+## Suggestions
+
+If you have any ideas for suggestions or other improvements to this guide,
+please let us know! I hope this was helpful. Please reach out if you are
+building an API that you might like to have `mgmt` consume!

docs/style-guide.md

@@ -67,6 +67,37 @@ Whenever a constant or function parameter is defined, try and have the safer or
 default value be the `zero` value. For example, instead of `const NoDanger`, use
 `const AllowDanger` so that the `false` value is the safe scenario.
 
+### Method receiver pointers
+
+You almost always want any method receivers to be declared on the pointer to the
+struct. There are only a few rare situations where this is not the case. This
+makes it easier to merge future changes that mutate the state without wondering
+why you now have two different copies of a struct. When you do need to copy a
+a struct, you can add a `Copy()` method to it. It's true that in many situations
+adding the pointer adds a small performance penalty, but we haven't found them
+to be significant in practice. If you do have a performance sensitive patch
+which benefits from skipping the pointer, please demonstrate this need with
+data first.
+
+#### Example
+
+```golang
+type Foo struct {
+	Whatever string
+	// ...
+}
+
+// Bar is implemented correctly as a pointer on Foo.
+func (obj *Foo) Bar(baz string) int {
+	// ...
+}
+
+// Bar is implemented *incorrectly* without a pointer to Foo.
+func (obj Foo) Bar(baz string) int {
+	// ...
+}
+```
+
 ### Method receiver naming
 
 [Contrary](https://github.com/golang/go/wiki/CodeReviewComments#receiver-names)

docs/util/metadata.go

@@ -0,0 +1,103 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+// Package util handles metadata for documentation generation.
+package util
+
+import (
+	"fmt"
+)
+
+var (
+	registeredResourceMetadata = make(map[string]*Metadata) // must initialize
+	registeredFunctionMetadata = make(map[string]*Metadata) // must initialize
+)
+
+// RegisterResource records the metadata for a resource of this kind.
+func RegisterResource(kind string, metadata *Metadata) error {
+	if _, exists := registeredResourceMetadata[kind]; exists {
+		return fmt.Errorf("metadata kind %s is already registered", kind)
+	}
+
+	registeredResourceMetadata[kind] = metadata
+
+	return nil
+}
+
+// LookupResource looks up the metadata for a resource of this kind.
+func LookupResource(kind string) (*Metadata, error) {
+	metadata, exists := registeredResourceMetadata[kind]
+	if !exists {
+		return nil, fmt.Errorf("not found")
+	}
+	return metadata, nil
+}
+
+// RegisterFunction records the metadata for a function of this name.
+func RegisterFunction(name string, metadata *Metadata) error {
+	if _, exists := registeredFunctionMetadata[name]; exists {
+		return fmt.Errorf("metadata named %s is already registered", name)
+	}
+
+	registeredFunctionMetadata[name] = metadata
+
+	return nil
+}
+
+// LookupFunction looks up the metadata for a function of this name.
+func LookupFunction(name string) (*Metadata, error) {
+	metadata, exists := registeredFunctionMetadata[name]
+	if !exists {
+		return nil, fmt.Errorf("not found")
+	}
+	return metadata, nil
+}
+
+// Metadata stores some additional information about the function or resource.
+// This is used to automatically generate documentation.
+type Metadata struct {
+	// Filename is the filename (without any base dir path) that this is in.
+	Filename string
+
+	// Typename is the string name of the main resource struct or function.
+	Typename string
+}
+
+// GetMetadata returns some metadata about the func. It can be called at any
+// time. This must not be named the same as the struct it's on or using it as an
+// anonymous embedded struct will stop us from being able to call this method.
+func (obj *Metadata) GetMetadata() *Metadata {
+	//if obj == nil { // TODO: Do I need this?
+	//	return nil
+	//}
+	return &Metadata{
+		Filename: obj.Filename,
+		Typename: obj.Typename,
+	}
+}

engine/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/autoedge_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/cmp.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/copy.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/doc.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/edge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/error.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/fs.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/actions.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -192,11 +192,15 @@ func (obj *Engine) Process(ctx context.Context, vertex pgraph.Vertex) error {
 
 	} else {
 		// run the CheckApply!
+		if obj.Debug {
 			obj.Logf("%s: CheckApply(%t)", res, !noop)
+		}
 		// if this fails, don't UpdateTimestamp()
 		checkOK, err = res.CheckApply(ctx, !noop)
+		if !checkOK && obj.Debug { // don't log on (checkOK == true)
 			obj.Logf("%s: CheckApply(%t): Return(%t, %s)", res, !noop, checkOK, engineUtil.CleanError(err))
 		}
+	}
 
 	if checkOK && err != nil { // should never return this way
 		return fmt.Errorf("%s: resource programming error: CheckApply(%t): %t, %+v", res, !noop, checkOK, err)

engine/graph/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/autoedge/autoedge.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -39,7 +39,7 @@ import (
 
 // AutoEdge adds the automatic edges to the graph.
 func AutoEdge(graph *pgraph.Graph, debug bool, logf func(format string, v ...interface{})) error {
-	logf("adding autoedges...")
+	logf("building...")
 
 	// initially get all of the autoedges to seek out all possible errors
 	var err error
@@ -63,7 +63,9 @@ func AutoEdge(graph *pgraph.Graph, debug bool, logf func(format string, v ...int
 			continue
 		}
 		if autoEdgeObj == nil {
+			if debug {
 				logf("no auto edges were found for: %s", res)
+			}
 			continue // next vertex
 		}
 		autoEdgeObjMap[res] = autoEdgeObj // save for next loop
@@ -86,9 +88,9 @@ func AutoEdge(graph *pgraph.Graph, debug bool, logf func(format string, v ...int
 				break // inner loop
 			}
 			if debug {
-				logf("autoedge: UIDS:")
+				logf("UIDS:")
 				for i, u := range uids {
-					logf("autoedge: UID%d: %v", i, u)
+					logf("UID%d: %v", i, u)
 				}
 			}
 
@@ -129,7 +131,7 @@ func addEdgesByMatchingUIDS(res engine.EdgeableRes, uids []engine.ResUID, graph
 				continue
 			}
 			if debug {
-				logf("autoedge: Match: %s with UID: %s", r, uid)
+				logf("match: %s with UID: %s", r, uid)
 			}
 			// we must match to an effective UID for the resource,
 			// that is to say, the name value of a res is a helpful
@@ -138,13 +140,13 @@ func addEdgesByMatchingUIDS(res engine.EdgeableRes, uids []engine.ResUID, graph
 			if UIDExistsInUIDs(uid, r.UIDs()) {
 				// add edge from: r -> res
 				if uid.IsReversed() {
-					txt := fmt.Sprintf("%s -> %s (autoedge)", r, res)
-					logf("autoedge: adding: %s", txt)
+					txt := fmt.Sprintf("%s -> %s", r, res)
+					logf("adding: %s", txt)
 					edge := &engine.Edge{Name: txt}
 					graph.AddEdge(r, res, edge)
 				} else { // edges go the "normal" way, eg: pkg resource
-					txt := fmt.Sprintf("%s -> %s (autoedge)", res, r)
-					logf("autoedge: adding: %s", txt)
+					txt := fmt.Sprintf("%s -> %s", res, r)
+					logf("adding: %s", txt)
 					edge := &engine.Edge{Name: txt}
 					graph.AddEdge(res, r, edge)
 				}

engine/graph/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/autogroup/autogroup.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/autogroup/autogroup_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/autogroup/base.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/autogroup/nonreachability.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/autogroup/util.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/engine.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -106,7 +106,8 @@ func (obj *Engine) Init() error {
 	if obj.Prefix == "" || obj.Prefix == "/" {
 		return fmt.Errorf("the prefix of `%s` is invalid", obj.Prefix)
 	}
-	if err := os.MkdirAll(obj.Prefix, 0770); err != nil {
+	// 0775 since we want children to be able to read this!
+	if err := os.MkdirAll(obj.Prefix, 0775); err != nil {
 		return errwrap.Wrapf(err, "can't create prefix")
 	}
 
@@ -224,7 +225,7 @@ func (obj *Engine) Commit() error {
 		statePrefix := fmt.Sprintf("%s/", path.Join(obj.statePrefix(), pathUID))
 
 		// don't create this unless it *will* be used
-		//if err := os.MkdirAll(statePrefix, 0770); err != nil {
+		//if err := os.MkdirAll(statePrefix, 0775); err != nil {
 		//	return errwrap.Wrapf(err, "can't create state prefix")
 		//}
 
@@ -268,7 +269,7 @@ func (obj *Engine) Commit() error {
 					obj.wlock.Unlock()
 				}()
 
-				if obj.Debug || true {
+				if obj.Debug {
 					obj.Logf("%s: Working...", v)
 				}
 				// contains the Watch and CheckApply loops

engine/graph/graph_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/refresh.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/reverse.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/semaphore.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/semaphore_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/sendrecv.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/state.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/graph/vardir.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -54,7 +54,8 @@ func (obj *State) varDir(extra string) (string, error) {
 
 	// an empty string at the end has no effect
 	p := fmt.Sprintf("%s/", path.Join(obj.Prefix, extra))
-	if err := os.MkdirAll(p, 0770); err != nil {
+	// 0775 since we want children to be able to read this!
+	if err := os.MkdirAll(p, 0775); err != nil {
 		return "", errwrap.Wrapf(err, "can't create prefix in: %s", p)
 	}
 

engine/graphqueryable.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/local/local.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -38,6 +38,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -54,7 +55,15 @@ type API struct {
 	Logf   func(format string, v ...interface{})
 
 	// Each piece of the API can take a handle here.
-	*Value
+	*Value // TODO: Rename to ValueImpl?
+
+	// VarDirImpl is the implementation for the VarDir API's. The API's are
+	// the collection of public methods that exist on this struct.
+	*VarDirImpl
+
+	// PoolImpl is the implementation for the Pool API's. The API's are the
+	// collection of public methods that exist on this struct.
+	*PoolImpl
 }
 
 // Init initializes the API before first use. It returns itself so it can be
@@ -67,6 +76,20 @@ func (obj *API) Init() *API {
 		Logf:   obj.Logf,
 	})
 
+	obj.VarDirImpl = &VarDirImpl{}
+	obj.VarDirImpl.Init(&VarDirInit{
+		Prefix: obj.Prefix,
+		Debug:  obj.Debug,
+		Logf:   obj.Logf,
+	})
+
+	obj.PoolImpl = &PoolImpl{}
+	obj.PoolImpl.Init(&PoolInit{
+		Prefix: obj.Prefix,
+		Debug:  obj.Debug,
+		Logf:   obj.Logf,
+	})
+
 	return obj
 }
 
@@ -332,3 +355,240 @@ func valueRemove(ctx context.Context, prefix, key string) error {
 	}
 	return nil // ignore not found errors
 }
+
+// VarDirInit are the init values that the VarDir API needs to work correctly.
+type VarDirInit struct {
+	Prefix string
+	Debug  bool
+	Logf   func(format string, v ...interface{})
+}
+
+// VarDirImpl is the implementation for the VarDir API's. The API's are the
+// collection of public methods that exist on this struct.
+type VarDirImpl struct {
+	init         *VarDirInit
+	mutex        *sync.Mutex
+	prefix       string
+	prefixExists bool // is it okay to use the prefix?
+}
+
+// Init runs some initialization code for the VarDir API.
+func (obj *VarDirImpl) Init(init *VarDirInit) {
+	obj.init = init
+	obj.mutex = &sync.Mutex{}
+	obj.prefix = fmt.Sprintf("%s/", path.Join(obj.init.Prefix, "vardir"))
+}
+
+// VarDir returns a directory rooted at the internal prefix.
+func (obj *VarDirImpl) VarDir(ctx context.Context, reldir string) (string, error) {
+	if strings.HasPrefix(reldir, "/") {
+		return "", fmt.Errorf("path must be relative")
+	}
+	if !strings.HasSuffix(reldir, "/") {
+		return "", fmt.Errorf("path must be a dir")
+	}
+	// NOTE: The above checks ensure we don't get either "" or "/" as input!
+
+	prefix, err := obj.getPrefix()
+	if err != nil {
+		return "", err
+	}
+
+	result := fmt.Sprintf("%s/", path.Join(prefix, reldir))
+
+	// TODO: Should we mkdir this?
+	obj.mutex.Lock()
+	defer obj.mutex.Unlock()
+	if err := os.MkdirAll(result, 0755); err != nil {
+		return "", err
+	}
+
+	return result, nil
+}
+
+// getPrefix gets the prefix dir to use, or errors if it can't make one. It
+// makes it on first use, and returns quickly from any future calls to it.
+func (obj *VarDirImpl) getPrefix() (string, error) {
+	// NOTE: Moving this mutex to just below the first early return, would
+	// be a benign race, but as it turns out, it's possible that a compiler
+	// would see this behaviour as "undefined" and things might not work as
+	// intended. It could perhaps be replaced with a sync/atomic primitive
+	// if we wanted better performance here.
+	obj.mutex.Lock()
+	defer obj.mutex.Unlock()
+
+	if obj.prefixExists { // former race read
+		return obj.prefix, nil
+	}
+
+	// MkdirAll instead of Mkdir because we have no idea if the parent
+	// local/ directory was already made yet or not. (If at all.) If path is
+	// already a directory, MkdirAll does nothing and returns nil. (Good!)
+	// TODO: I hope MkdirAll is thread-safe on path creation in case another
+	// future local API tries to make the base (parent) directory too!
+	if err := os.MkdirAll(obj.prefix, 0755); err != nil {
+		return "", err
+	}
+	obj.prefixExists = true // former race write
+
+	return obj.prefix, nil
+}
+
+// PoolInit are the init values that the Pool API needs to work correctly.
+type PoolInit struct {
+	Prefix string
+	Debug  bool
+	Logf   func(format string, v ...interface{})
+}
+
+// PoolConfig configures how the Pool operates.
+// XXX: These are not implemented yet.
+type PoolConfig struct {
+	// Expiry specifies that we expire old values that have not been read
+	// for this many seconds. Zero disables this and they never expire.
+	Expiry int64 // TODO: or time.Time ?
+
+	// Random lets you allocate a random integer instead of sequential ones.
+	Random bool
+
+	// Max specifies the maximum integer to allocate.
+	Max int
+}
+
+// PoolImpl is the implementation for the Pool API's. The API's are the
+// collection of public methods that exist on this struct.
+type PoolImpl struct {
+	init         *PoolInit
+	mutex        *sync.Mutex
+	prefix       string
+	prefixExists bool // is it okay to use the prefix?
+}
+
+// Init runs some initialization code for the Pool API.
+func (obj *PoolImpl) Init(init *PoolInit) {
+	obj.init = init
+	obj.mutex = &sync.Mutex{}
+	obj.prefix = fmt.Sprintf("%s/", path.Join(obj.init.Prefix, "pool"))
+}
+
+// Pool returns a unique integer from a pool of numbers. Within a given
+// namespace, it returns the same integer for a given name. It is a simple
+// mechanism to allocate numbers to different inputs when we don't have a
+// hashing alternative. It does not allocate zero.
+func (obj *PoolImpl) Pool(ctx context.Context, namespace, uid string, config *PoolConfig) (int, error) {
+	if namespace == "" {
+		return 0, fmt.Errorf("namespace is empty")
+	}
+	if strings.Contains(namespace, "/") {
+		return 0, fmt.Errorf("namespace contains slash")
+	}
+	if uid == "" {
+		return 0, fmt.Errorf("uid is empty")
+	}
+	if strings.Contains(uid, "/") {
+		return 0, fmt.Errorf("uid contains slash")
+	}
+
+	prefix, err := obj.getPrefix()
+	if err != nil {
+		return 0, err
+	}
+
+	dir := fmt.Sprintf("%s/", path.Join(prefix, namespace))
+	file := fmt.Sprintf("%s.uid", path.Join(dir, uid)) // file
+
+	// TODO: Run clean up funcs here to get rid of any stale/expired values.
+	// TODO: This will happen based on the future config options we build...
+
+	obj.mutex.Lock()
+	defer obj.mutex.Unlock()
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return 0, err
+	}
+
+	fn := func(p string) (int, error) {
+		b, err := os.ReadFile(p)
+		if err != nil && !os.IsNotExist(err) {
+			return 0, err // real error
+		}
+		if err != nil {
+			return 0, nil // absent!
+		}
+
+		// File exists!
+		d, err := strconv.Atoi(strings.TrimSpace(string(b)))
+		if err != nil {
+			// Someone put corrupt data in a uid file.
+			return 0, err // real error
+		}
+		return d, nil // value already allocated!
+	}
+
+	d, err := fn(file)
+	if err != nil {
+		return 0, err // real error
+	}
+	if d != 0 {
+		return d, nil // Value already allocated! We're done early.
+	}
+
+	// Not found, so find the max. (0 without error means not found!)
+
+	files, err := os.ReadDir(dir) // ([]os.DirEntry, error)
+	if err != nil {
+		return 0, err // real error
+	}
+
+	m := 0
+	for _, f := range files {
+		if f.IsDir() {
+			continue // unexpected!
+		}
+		d, err := fn(path.Join(dir, f.Name()))
+		if err != nil {
+			return 0, err // real error
+		}
+		if d == 0 {
+			// Must be someone deleting files without our mutex!
+			return 0, fmt.Errorf("unexpected missing file")
+		}
+
+		m = max(m, d)
+	}
+
+	m++                                    // increment
+	data := []byte(fmt.Sprintf("%d\n", m)) // it's polite to end with \n
+	if err := os.WriteFile(file, data, 0600); err != nil {
+		return 0, err
+	}
+
+	return m, nil
+}
+
+// getPrefix gets the prefix dir to use, or errors if it can't make one. It
+// makes it on first use, and returns quickly from any future calls to it.
+func (obj *PoolImpl) getPrefix() (string, error) {
+	// NOTE: Moving this mutex to just below the first early return, would
+	// be a benign race, but as it turns out, it's possible that a compiler
+	// would see this behaviour as "undefined" and things might not work as
+	// intended. It could perhaps be replaced with a sync/atomic primitive
+	// if we wanted better performance here.
+	obj.mutex.Lock()
+	defer obj.mutex.Unlock()
+
+	if obj.prefixExists { // former race read
+		return obj.prefix, nil
+	}
+
+	// MkdirAll instead of Mkdir because we have no idea if the parent
+	// local/ directory was already made yet or not. (If at all.) If path is
+	// already a directory, MkdirAll does nothing and returns nil. (Good!)
+	// TODO: I hope MkdirAll is thread-safe on path creation in case another
+	// future local API tries to make the base (parent) directory too!
+	if err := os.MkdirAll(obj.prefix, 0755); err != nil {
+		return "", err
+	}
+	obj.prefixExists = true // former race write
+
+	return obj.prefix, nil
+}

engine/local/local_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/metaparams.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -52,6 +52,7 @@ var DefaultMetaParams = &MetaParams{
 	//Sema:  []string{},
 	Rewatch: false,
 	Realize: false, // true would be more awesome, but unexpected for users
+	Dollar:  false,
 }
 
 // MetaRes is the interface a resource must implement to support meta params.
@@ -132,6 +133,13 @@ type MetaParams struct {
 	// the resource is blocked because of a failed pre-requisite resource.
 	// XXX: Not implemented!
 	Realize bool `yaml:"realize"`
+
+	// Dollar allows you to name a resource to start with the dollar
+	// character. We don't allow this by default since it's probably not
+	// needed, and is more likely to be a typo where the user forgot to
+	// interpolate a variable name. In the rare case when it's needed, you
+	// can disable that check with this meta param.
+	Dollar bool `yaml:"dollar"`
 }
 
 // Cmp compares two AutoGroupMeta structs and determines if they're equivalent.
@@ -178,6 +186,9 @@ func (obj *MetaParams) Cmp(meta *MetaParams) error {
 	if obj.Realize != meta.Realize {
 		return fmt.Errorf("values for Realize are different")
 	}
+	if obj.Dollar != meta.Dollar {
+		return fmt.Errorf("values for Dollar are different")
+	}
 
 	return nil
 }
@@ -218,6 +229,7 @@ func (obj *MetaParams) Copy() *MetaParams {
 		Sema:    sema,
 		Rewatch: obj.Rewatch,
 		Realize: obj.Realize,
+		Dollar:  obj.Dollar,
 	}
 }
 

engine/metaparams_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/refresh.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -33,7 +33,12 @@ import (
 	"context"
 	"encoding/gob"
 	"fmt"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"strings"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/engine/local"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util/errwrap"
@@ -41,6 +46,12 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+const (
+	// ResourcesRelDir is the path where the resources are kept, relative to
+	// the main source code root.
+	ResourcesRelDir = "engine/resources/"
+)
+
 // TODO: should each resource be a sub-package?
 var registeredResources = map[string]func() Res{}
 
@@ -56,6 +67,23 @@ func RegisterResource(kind string, fn func() Res) {
 	}
 	gob.Register(f)
 	registeredResources[kind] = fn
+
+	// Additional metadata for documentation generation!
+	_, filename, _, ok := runtime.Caller(1)
+	if !ok {
+		panic(fmt.Sprintf("could not locate resource filename for %s", kind))
+	}
+	sp := strings.Split(reflect.TypeOf(f).String(), ".")
+	if len(sp) != 2 {
+		panic(fmt.Sprintf("could not parse resource struct name for %s", kind))
+	}
+
+	if err := docsUtil.RegisterResource(kind, &docsUtil.Metadata{
+		Filename: filepath.Base(filename),
+		Typename: sp[1],
+	}); err != nil {
+		panic(fmt.Sprintf("could not register resource metadata for %s", kind))
+	}
 }
 
 // RegisteredResourcesNames returns the kind of the registered resources.
@@ -272,6 +300,12 @@ func Validate(res Res) error {
 		return errwrap.Wrapf(err, "the Res has an invalid meta param")
 	}
 
+	// TODO: pull dollar prefix from a constant
+	// This catches typos where the user meant to use ${var} interpolation.
+	if !res.MetaParams().Dollar && strings.HasPrefix(res.Name(), "$") {
+		return fmt.Errorf("the Res name starts with a $")
+	}
+
 	return res.Validate()
 }
 

engine/resources/augeas.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -206,7 +206,6 @@ func (obj *AugeasRes) checkApplySet(ctx context.Context, apply bool, ag *augeas.
 
 // CheckApply method for Augeas resource.
 func (obj *AugeasRes) CheckApply(ctx context.Context, apply bool) (bool, error) {
-	obj.init.Logf("CheckApply: %s", obj.File)
 	// By default we do not set any option to augeas, we use the defaults.
 	opts := augeas.None
 	if obj.Lens != "" {

engine/resources/aws_ec2.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -638,7 +638,7 @@ func (obj *AwsEc2Res) snsWatch(ctx context.Context) error {
 
 // CheckApply method for AwsEc2 resource.
 func (obj *AwsEc2Res) CheckApply(ctx context.Context, apply bool) (bool, error) {
-	obj.init.Logf("CheckApply(%t)", apply)
+	obj.init.Logf("CheckApply(%t)", apply) // XXX: replace with logf on change
 
 	// find the instance we need to check
 	instance, err := describeInstanceByName(obj.client, obj.prependName())

engine/resources/bmc_power.go

@@ -0,0 +1,466 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package resources
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/traits"
+
+	bmclib "github.com/bmc-toolbox/bmclib/v2"
+	"github.com/bmc-toolbox/bmclib/v2/providers/rpc"
+)
+
+func init() {
+	engine.RegisterResource("bmc:power", func() engine.Res { return &BmcPowerRes{} })
+}
+
+const (
+	// DefaultBmcPowerPort is the default port we try to connect on.
+	DefaultBmcPowerPort = 443
+
+	// BmcDriverSecureSuffix is the magic char we append to a driver name to
+	// specify we want the SSL/TLS variant.
+	BmcDriverSecureSuffix = "s"
+
+	// BmcDriverRPC is the RPC driver.
+	BmcDriverRPC = "rpc"
+
+	// BmcDriverGofish is the gofish driver.
+	BmcDriverGofish = "gofish"
+)
+
+// BmcPowerRes is a resource that manages power state of a BMC. This is usually
+// used for turning computers on and off. The name value can be a big URL string
+// in the form: `driver://user:pass@hostname:port` for example you may see:
+// gofishs://ADMIN:hunter2@127.0.0.1:8800 to use the "https" variant of the
+// gofish driver.
+//
+// NOTE: New drivers should either not end in "s" or at least not be identical
+// to the name of another driver an "s" is added or removed to the end.
+type BmcPowerRes struct {
+	traits.Base // add the base methods without re-implementation
+
+	init *engine.Init
+
+	// Hostname to connect to. If not specified, we parse this from the
+	// Name.
+	Hostname string `lang:"hostname" yaml:"hostname"`
+
+	// Port to connect to. If not specified, we parse this from the Name.
+	Port int `lang:"port" yaml:"port"`
+
+	// Username to use to connect. If not specified, we parse this from the
+	// Name.
+	// TODO: If the Username field is not set, should we parse from the
+	// Name? It's not really part of the BMC unique identifier so maybe we
+	// shouldn't use that.
+	Username string `lang:"username" yaml:"username"`
+
+	// Password to use to connect. We do NOT parse this from the Name unless
+	// you set InsecurePassword to true.
+	// XXX: Use mgmt magic credentials in the future.
+	Password string `lang:"password" yaml:"password"`
+
+	// InsecurePassword can be set to true to allow a password in the Name.
+	InsecurePassword bool `lang:"insecure_password" yaml:"insecure_password"`
+
+	// Driver to use, such as: "gofish" or "rpc". This is a different
+	// concept than the "bmclib" driver vs provider distinction. Here we
+	// just statically pick what we're using without any magic. If not
+	// specified, we parse this from the Name scheme. If this ends with an
+	// extra "s" then we use https instead of http.
+	Driver string `lang:"driver" yaml:"driver"`
+
+	// State of machine power. Can be "on" or "off".
+	State string `lang:"state" yaml:"state"`
+
+	driver string
+	scheme string
+}
+
+// validDriver determines if we are using a valid drive. This does not include
+// the magic "s" bits. This function need to be expanded as we support more
+// drivers.
+func (obj *BmcPowerRes) validDriver(driver string) error {
+	if driver == BmcDriverRPC {
+		return nil
+	}
+	if driver == BmcDriverGofish {
+		return nil
+	}
+
+	return fmt.Errorf("unknown driver: %s", driver)
+}
+
+// getHostname returns the hostname that we want to connect to. If the Hostname
+// field is set, we use that, otherwise we parse from the Name.
+func (obj *BmcPowerRes) getHostname() string {
+	if obj.Hostname != "" {
+		return obj.Hostname
+	}
+
+	u, err := url.Parse(obj.Name())
+	if err != nil || u == nil {
+		return ""
+	}
+
+	// SplitHostPort splits a network address of the form "host:port",
+	// "host%zone:port", "[host]:port" or "[host%zone]:port" into host or
+	// host%zone and port.
+	host, port, err := net.SplitHostPort(u.Host)
+	if err != nil {
+		return u.Host // must be a naked hostname or ip w/o port
+	}
+	_ = port
+
+	return host
+}
+
+// getPort returns the port that we want to connect to. If the Port field is
+// set, we use that, otherwise we parse from the Name.
+//
+// NOTE: We return a string since all the bmclib things usually expect a string,
+// but if that gets fixed we should return an int here instead.
+func (obj *BmcPowerRes) getPort() string {
+	if obj.Port != 0 {
+		return strconv.Itoa(obj.Port)
+	}
+
+	u, err := url.Parse(obj.Name())
+	if err != nil || u == nil {
+		return ""
+	}
+
+	// SplitHostPort splits a network address of the form "host:port",
+	// "host%zone:port", "[host]:port" or "[host%zone]:port" into host or
+	// host%zone and port.
+	host, port, err := net.SplitHostPort(u.Host)
+	if err != nil {
+		return strconv.Itoa(DefaultBmcPowerPort) // default port
+	}
+	_ = host
+
+	return port
+}
+
+// getUsername returns the username that we want to connect with. If the
+// Username field is set, we use that, otherwise we parse from the Name.
+// TODO: If the Username field is not set, should we parse from the Name? It's
+// not really part of the BMC unique identifier so maybe we shouldn't use that.
+func (obj *BmcPowerRes) getUsername() string {
+	if obj.Username != "" {
+		return obj.Username
+	}
+
+	u, err := url.Parse(obj.Name())
+	if err != nil || u == nil || u.User == nil {
+		return ""
+	}
+
+	return u.User.Username()
+}
+
+// getPassword returns the password that we want to connect with.
+// XXX: Use mgmt magic credentials in the future.
+func (obj *BmcPowerRes) getPassword() string {
+	if obj.Password != "" || !obj.InsecurePassword {
+		return obj.Password
+	}
+	// NOTE: We don't look at any password string from the name unless the
+	// InsecurePassword field is true.
+
+	u, err := url.Parse(obj.Name())
+	if err != nil || u == nil || u.User == nil {
+		return ""
+	}
+
+	password, ok := u.User.Password()
+	if !ok {
+		return ""
+	}
+
+	return password
+}
+
+// getRawDriver returns the raw magic driver string. If the Driver field is set,
+// we use that, otherwise we parse from the Name. This version may include the
+// magic "s" at the end.
+func (obj *BmcPowerRes) getRawDriver() string {
+	if obj.Driver != "" {
+		return obj.Driver
+	}
+
+	u, err := url.Parse(obj.Name())
+	if err != nil || u == nil {
+		return ""
+	}
+
+	return u.Scheme
+}
+
+// getDriverAndScheme figures out which driver and scheme we want to use.
+func (obj *BmcPowerRes) getDriverAndScheme() (string, string, error) {
+	driver := obj.getRawDriver()
+	err := obj.validDriver(driver)
+	if err == nil {
+		return driver, "http", nil
+	}
+
+	driver = strings.TrimSuffix(driver, BmcDriverSecureSuffix)
+	if err := obj.validDriver(driver); err == nil {
+		return driver, "https", nil
+	}
+
+	return "", "", err // return the first error
+}
+
+// getDriver returns the actual driver that we want to connect with. If the
+// Driver field is set, we use that, otherwise we parse from the Name. This
+// version does NOT include the magic "s" at the end.
+func (obj *BmcPowerRes) getDriver() string {
+	return obj.driver
+}
+
+// getScheme figures out which scheme we want to use.
+func (obj *BmcPowerRes) getScheme() string {
+	return obj.scheme
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *BmcPowerRes) Default() engine.Res {
+	return &BmcPowerRes{}
+}
+
+// Validate if the params passed in are valid data.
+func (obj *BmcPowerRes) Validate() error {
+	// XXX: Force polling until we have real events...
+	if obj.MetaParams().Poll == 0 {
+		return fmt.Errorf("events are not yet supported, use polling")
+	}
+
+	if obj.getHostname() == "" {
+		return fmt.Errorf("need a Hostname")
+	}
+	//if obj.getUsername() == "" {
+	//	return fmt.Errorf("need a Username")
+	//}
+
+	if obj.getRawDriver() == "" {
+		return fmt.Errorf("need a Driver")
+	}
+	if _, _, err := obj.getDriverAndScheme(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *BmcPowerRes) Init(init *engine.Init) error {
+	obj.init = init // save for later
+
+	driver, scheme, err := obj.getDriverAndScheme()
+	if err != nil {
+		// programming error (we checked in Validate)
+		return err
+	}
+	obj.driver = driver
+	obj.scheme = scheme
+
+	return nil
+}
+
+// Cleanup is run by the engine to clean up after the resource is done.
+func (obj *BmcPowerRes) Cleanup() error {
+	return nil
+}
+
+// client builds the bmclib client. The API to build it is complicated.
+func (obj *BmcPowerRes) client() *bmclib.Client {
+	// NOTE: The bmclib API is weird, you can't put the port in this string!
+	u := fmt.Sprintf("%s://%s", obj.getScheme(), obj.getHostname())
+
+	uPort := u
+	if p := obj.getPort(); p != "" {
+		uPort = u + ":" + p
+	}
+
+	opts := []bmclib.Option{}
+
+	if obj.getDriver() == BmcDriverRPC {
+		opts = append(opts, bmclib.WithRPCOpt(rpc.Provider{
+			// NOTE: The main API cannot take a port, but here we do!
+			ConsumerURL: uPort,
+		}))
+	}
+
+	if p := obj.getPort(); p != "" {
+		switch obj.getDriver() {
+		case BmcDriverRPC:
+			// TODO: ???
+
+		case BmcDriverGofish:
+			// XXX: Why doesn't this accept an int?
+			opts = append(opts, bmclib.WithRedfishPort(p))
+
+		//case BmcDriverOpenbmc:
+		//	// XXX: Why doesn't this accept an int?
+		//	opts = append(opts, openbmc.WithPort(p))
+
+		default:
+			// TODO: error or pass through?
+			obj.init.Logf("unhandled driver: %s", obj.getDriver())
+		}
+	}
+
+	client := bmclib.NewClient(u, obj.getUsername(), obj.Password, opts...)
+
+	if obj.getDriver() != "" && obj.getDriver() != BmcDriverRPC {
+		client = client.For(obj.getDriver()) // limit to this provider
+	}
+
+	return client
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *BmcPowerRes) Watch(ctx context.Context) error {
+	obj.init.Running() // when started, notify engine that we're running
+
+	select {
+	case <-ctx.Done(): // closed by the engine to signal shutdown
+	}
+
+	//obj.init.Event() // notify engine of an event (this can block)
+
+	return nil
+}
+
+// CheckApply method for BmcPower resource. Does nothing, returns happy!
+func (obj *BmcPowerRes) CheckApply(ctx context.Context, apply bool) (bool, error) {
+
+	client := obj.client()
+
+	if err := client.Open(ctx); err != nil {
+		return false, err
+	}
+	defer client.Close(ctx) // (err error)
+
+	if obj.init.Debug {
+		obj.init.Logf("connected ok")
+	}
+
+	state, err := client.GetPowerState(ctx)
+	if err != nil {
+		return false, err
+	}
+	state = strings.ToLower(state) // normalize
+	obj.init.Logf("get state: %s", state)
+
+	if !apply {
+		return false, nil
+	}
+
+	if obj.State == state {
+		return true, nil
+	}
+
+	// TODO: should this be "On" and "Off"? Does case matter?
+	ok, err := client.SetPowerState(ctx, obj.State)
+	if err != nil {
+		return false, err
+	}
+	if !ok {
+		// TODO: When is this ever false?
+	}
+	obj.init.Logf("set state: %s", obj.State)
+
+	return false, nil
+}
+
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *BmcPowerRes) Cmp(r engine.Res) error {
+	// we can only compare BmcPowerRes to others of the same resource kind
+	res, ok := r.(*BmcPowerRes)
+	if !ok {
+		return fmt.Errorf("not a %s", obj.Kind())
+	}
+
+	if obj.Hostname != res.Hostname {
+		return fmt.Errorf("the Hostname differs")
+	}
+	if obj.Port != res.Port {
+		return fmt.Errorf("the Port differs")
+	}
+	if obj.Username != res.Username {
+		return fmt.Errorf("the Username differs")
+	}
+	if obj.Password != res.Password {
+		return fmt.Errorf("the Password differs")
+	}
+	if obj.InsecurePassword != res.InsecurePassword {
+		return fmt.Errorf("the InsecurePassword differs")
+	}
+
+	if obj.Driver != res.Driver {
+		return fmt.Errorf("the Driver differs")
+	}
+	if obj.State != res.State {
+		return fmt.Errorf("the State differs")
+	}
+
+	return nil
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
+func (obj *BmcPowerRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes BmcPowerRes // indirection to avoid infinite recursion
+
+	def := obj.Default()          // get the default
+	res, ok := def.(*BmcPowerRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to BmcPowerRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = BmcPowerRes(raw) // restore from indirection with type conversion!
+	return nil
+}

engine/resources/config_etcd.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/consul_kv.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -27,6 +27,8 @@
 // additional permission if he deems it necessary to achieve the goals of this
 // additional permission.
 
+//go:build !noconsul
+
 package resources
 
 import (

engine/resources/consul_kv_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -27,6 +27,8 @@
 // additional permission if he deems it necessary to achieve the goals of this
 // additional permission.
 
+//go:build !root || !noconsul
+
 package resources
 
 import (

engine/resources/cron.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -36,7 +36,6 @@ import (
 	"os/user"
 	"path"
 	"strings"
-	"time"
 
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/traits"
@@ -76,10 +75,6 @@ const (
 	// in 'man systemd-timer', and whose format is a time span as defined in
 	// 'man systemd-time'.
 	OnUnitInactiveSec = "OnUnitInactiveSec"
-
-	// ctxTimeout is the delay, in seconds, before the calls to restart or stop
-	// the systemd unit will error due to timeout.
-	ctxTimeout = 30
 )
 
 func init() {
@@ -104,6 +99,11 @@ type CronRes struct {
 	// State must be 'exists' or 'absent'.
 	State string `lang:"state" yaml:"state"`
 
+	// Startup specifies what should happen on startup. Values can be:
+	// enabled, disabled, and undefined (empty string). We default to
+	// enabled.
+	Startup string `lang:"startup" yaml:"startup"`
+
 	// Session, if true, creates the timer as the current user, rather than
 	// root. The service it points to must also be a user unit. It defaults
 	// to false.
@@ -154,6 +154,7 @@ type CronRes struct {
 func (obj *CronRes) Default() engine.Res {
 	return &CronRes{
 		State:             "exists",
+		Startup:           "enabled",
 		RemainAfterElapse: true,
 	}
 }
@@ -188,6 +189,9 @@ func (obj *CronRes) Validate() error {
 	if obj.State != "absent" && obj.State != "exists" {
 		return fmt.Errorf("state must be 'absent' or 'exists'")
 	}
+	if obj.Startup != "enabled" && obj.Startup != "disabled" && obj.Startup != "" {
+		return fmt.Errorf("startup must be either `enabled` or `disabled` or undefined")
+	}
 
 	// validate trigger
 	if obj.State == "absent" && obj.Trigger == "" {
@@ -264,12 +268,12 @@ func (obj *CronRes) Watch(ctx context.Context) error {
 	args := []string{}
 	args = append(args, "type='signal'")
 	args = append(args, "interface='org.freedesktop.systemd1.Manager'")
-	args = append(args, "eavesdrop='true'")
+	//args = append(args, "eavesdrop='true'") // XXX: not allowed anymore?
 	args = append(args, fmt.Sprintf("arg2='%s.timer'", obj.Name()))
 
 	// match dbus messsages
 	if call := bus.BusObject().Call(engineUtil.DBusAddMatch, 0, strings.Join(args, ",")); call.Err != nil {
-		return err
+		return call.Err
 	}
 	defer bus.BusObject().Call(engineUtil.DBusRemoveMatch, 0, args) // ignore the error
 
@@ -390,14 +394,10 @@ func (obj *CronRes) unitCheckApply(ctx context.Context, apply bool) (bool, error
 	}
 
 	// systemctl daemon-reload
-	if err := conn.Reload(); err != nil {
+	if err := conn.ReloadContext(ctx); err != nil {
 		return false, errwrap.Wrapf(err, "error reloading daemon")
 	}
 
-	// context for stopping/restarting the unit
-	ctx, cancel := context.WithTimeout(ctx, ctxTimeout*time.Second)
-	defer cancel()
-
 	// godbus connection for stopping/restarting the unit
 	if obj.Session {
 		godbusConn, err = util.SessionBusPrivateUsable()
@@ -409,6 +409,18 @@ func (obj *CronRes) unitCheckApply(ctx context.Context, apply bool) (bool, error
 	}
 	defer godbusConn.Close()
 
+	// We probably always want to enable this...
+	svc := fmt.Sprintf("%s.timer", obj.Name()) // systemd name
+	files := []string{svc}                     // the svc represented in a list
+	if obj.Startup == "enabled" {
+		_, _, err = conn.EnableUnitFilesContext(ctx, files, false, true)
+	} else if obj.Startup == "disabled" {
+		_, err = conn.DisableUnitFilesContext(ctx, files, false)
+	}
+	if err != nil {
+		return false, errwrap.Wrapf(err, "unable to change startup status")
+	}
+
 	// stop or restart the unit
 	if obj.State == "absent" {
 		return false, engineUtil.StopUnit(ctx, godbusConn, fmt.Sprintf("%s.timer", obj.Name()))
@@ -426,6 +438,9 @@ func (obj *CronRes) Cmp(r engine.Res) error {
 	if obj.State != res.State {
 		return fmt.Errorf("state differs: %s vs %s", obj.State, res.State)
 	}
+	if obj.Startup != res.Startup {
+		return fmt.Errorf("the Startup differs")
+	}
 	if obj.Trigger != res.Trigger {
 		return fmt.Errorf("trigger differs: %s vs %s", obj.Trigger, res.Trigger)
 	}

engine/resources/deploy_tar.go

@@ -0,0 +1,528 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package resources
+
+import (
+	"archive/tar"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/traits"
+	"github.com/purpleidea/mgmt/util/errwrap"
+	"github.com/purpleidea/mgmt/util/recwatch"
+
+	"github.com/spf13/afero"
+)
+
+func init() {
+	engine.RegisterResource("deploy:tar", func() engine.Res { return &DeployTar{} })
+}
+
+// DeployTar is a resource that archives a deploy filesystem using tar, thus
+// combining them into a single file. The name of the resource is the path to
+// the resultant archive file. The input comes from the current deploy. This
+// uses hashes to determine if something was changed, so as a result, this may
+// not be suitable if you can create a sha256 hash collision.
+// TODO: support send/recv to send the output instead of writing to a file?
+// TODO: This resource is very similar to the tar resource. Update that one if
+// this changes, or consider porting this to use that as a composite resource.
+// TODO: consider using a `deploy.get_archive()` function to make a .tar, and a
+// file resource to store those contents on disk with whatever mode we want...
+type DeployTar struct {
+	traits.Base // add the base methods without re-implementation
+
+	init *engine.Init
+
+	// Path, which defaults to the name if not specified, represents the
+	// destination path for the compressed file being created. It must be an
+	// absolute path, and as a result must start with a slash. Since it is a
+	// file, it must not end with a slash.
+	Path string `lang:"path" yaml:"path"`
+
+	// Format is the header format to use. If you change this, then the
+	// file will get rearchived. The strange thing is that it seems the
+	// header format is stored for each individual file. The available
+	// values are: const.res.tar.format.unknown, const.res.tar.format.ustar,
+	// const.res.tar.format.pax, and const.res.tar.format.gnu which have
+	// values of 0, 2, 4, and 8 respectively.
+	Format int `lang:"format" yaml:"format"`
+
+	// SendOnly specifies that we don't write the file to disk, and as a
+	// result, the output is only be accessible by the send/recv mechanism.
+	// TODO: Rename this?
+	// TODO: Not implemented
+	//SendOnly bool `lang:"sendonly" yaml:"sendonly"`
+
+	// varDirPathInput is the path we use to store the content hash.
+	varDirPathInput string
+
+	// varDirPathOutput is the path we use to store the output file hash.
+	varDirPathOutput string
+}
+
+// getPath returns the actual path to use for this resource. It computes this
+// after analysis of the Path and Name.
+func (obj *DeployTar) getPath() string {
+	p := obj.Path
+	if obj.Path == "" { // use the name as the path default if missing
+		p = obj.Name()
+	}
+	return p
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *DeployTar) Default() engine.Res {
+	return &DeployTar{
+		Format: int(tar.FormatUnknown), // TODO: will this let it auto-choose?
+	}
+}
+
+// Validate if the params passed in are valid data.
+func (obj *DeployTar) Validate() error {
+	if obj.getPath() == "" {
+		return fmt.Errorf("path is empty")
+	}
+	if !strings.HasPrefix(obj.getPath(), "/") {
+		return fmt.Errorf("path must be absolute")
+	}
+	if strings.HasSuffix(obj.getPath(), "/") {
+		return fmt.Errorf("path must not end with a slash")
+	}
+
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *DeployTar) Init(init *engine.Init) error {
+	obj.init = init // save for later
+
+	dir, err := obj.init.VarDir("")
+	if err != nil {
+		return errwrap.Wrapf(err, "could not get VarDir in Init()")
+	}
+	// return unique files
+	obj.varDirPathInput = path.Join(dir, "input.sha256")
+	obj.varDirPathOutput = path.Join(dir, "output.sha256")
+
+	return nil
+}
+
+// Cleanup is run by the engine to clean up after the resource is done.
+func (obj *DeployTar) Cleanup() error {
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *DeployTar) Watch(ctx context.Context) error {
+	recurse := false // single (output) file
+	recWatcher, err := recwatch.NewRecWatcher(obj.getPath(), recurse)
+	if err != nil {
+		return err
+	}
+	defer recWatcher.Close()
+
+	obj.init.Running() // when started, notify engine that we're running
+
+	var send = false // send event?
+	for {
+		select {
+		case event, ok := <-recWatcher.Events():
+			if !ok { // channel shutdown
+				// TODO: Should this be an error? Previously it
+				// was a `return nil`, and i'm not sure why...
+				//return nil
+				return fmt.Errorf("unexpected close")
+			}
+			if err := event.Error; err != nil {
+				return errwrap.Wrapf(err, "unknown %s watcher error", obj)
+			}
+			if obj.init.Debug { // don't access event.Body if event.Error isn't nil
+				obj.init.Logf("event(%s): %v", event.Body.Name, event.Body.Op)
+			}
+			send = true
+
+		case <-ctx.Done(): // closed by the engine to signal shutdown
+			return nil
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			obj.init.Event() // notify engine of an event (this can block)
+		}
+	}
+}
+
+// CheckApply checks the resource state and applies the resource if the bool
+// input is true. It returns error info and if the state check passed or not.
+// This is where we actually do the archiving into a tar file work when needed.
+func (obj *DeployTar) CheckApply(ctx context.Context, apply bool) (bool, error) {
+	uri := obj.init.World.URI() // request each time to ensure it's fresh!
+
+	filesystem, err := obj.init.World.Fs(uri) // open the remote file system
+	if err != nil {
+		return false, errwrap.Wrapf(err, "can't load code from file system `%s`", uri)
+	}
+
+	h1, err := obj.hashFile(obj.getPath()) // output
+	if err != nil {
+		return false, err
+	}
+
+	h2, err := obj.readHashFile(obj.varDirPathOutput, true)
+	if err != nil {
+		return false, err
+	}
+
+	i1 := ""
+	i1 = obj.formatPrefix() + "\n" // add the prefix so it is considered
+
+	// TODO: use standard filesystem API's when we can make them work!
+	//fsys := afero.NewIOFS(filesystem)
+
+	if err := afero.Walk(filesystem, "/", func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			if path == "/" { // special case for root
+				i1 += path + "|" + "\n"
+				return nil
+			}
+			// hash the dir itself too (eg: empty dirs!)
+			i1 += path + "/" + "|" + "\n"
+			return nil
+		}
+
+		h, err := obj.hashFileAferoFs(filesystem, path)
+		if err != nil {
+			return err
+		}
+		i1 += path + "|" + h + "\n"
+		return nil
+
+	}); err != nil {
+		return false, err
+	}
+
+	i2, err := obj.readHashFile(obj.varDirPathInput, false)
+	if err != nil {
+		return false, err
+	}
+
+	// We're cheating by computing this before we know if we errored!
+	inputMatches := i1 == i2
+	outputMatches := h1 == h2
+	if err == nil && inputMatches && outputMatches {
+		// If the two hashes match, we assume that the file is correct!
+		// The file has to also exist of course...
+		return true, nil
+	}
+
+	if !apply {
+		return false, nil
+	}
+
+	fail := true // assume we have a failure
+
+	defer func() {
+		if !fail {
+			return
+		}
+		// Don't leave a partial file lying around...
+		obj.init.Logf("removing partial tar file")
+		err := os.Remove(obj.getPath())
+		if err == nil || os.IsNotExist(err) {
+			return
+		}
+		obj.init.Logf("error removing corrupt tar file: %v", err)
+	}()
+
+	// FIXME: Do we instead want to write to a tmp file and do a move once
+	// we finish writing to be atomic here and avoid partial corrupt files?
+	// FIXME: Add a param called Atomic to specify that behaviour. It's
+	// instant so that might be preferred as it might generate fewer events,
+	// but there's a chance it's copying from obj.init.VarDir() to a
+	// different filesystem.
+	outputFile, err := os.Create(obj.getPath()) // io.Writer
+	if err != nil {
+		return false, err
+	}
+	//defer outputFile.Sync() // not needed?
+	defer outputFile.Close()
+
+	hash := sha256.New()
+
+	// Write to both to avoid needing to wait for fsync to calculate hash!
+	multiWriter := io.MultiWriter(outputFile, hash)
+
+	tarWriter := tar.NewWriter(multiWriter) // (*tar.Writer, error)
+	defer tarWriter.Close()                 // Might as well always close if we error early!
+
+	// TODO: formerly tarWriter.AddFS(fsys) // buggy!
+	if err := obj.addAferoFs(tarWriter, filesystem); err != nil {
+		return false, errwrap.Wrapf(err, "error writing fs")
+	}
+
+	// NOTE: Must run this before hashing so that it includes the footer!
+	if err := tarWriter.Close(); err != nil {
+		return false, err
+	}
+	sha256sum := hex.EncodeToString(hash.Sum(nil))
+
+	// TODO: add better logging counts if we can see tarWriter.AddFs too!
+	//obj.init.Logf("wrote %d files into archive", ?)
+	obj.init.Logf("wrote tar archive")
+
+	// After tar is successfully written, store the hashed input result.
+	if !inputMatches {
+		if err := os.WriteFile(obj.varDirPathInput, []byte(i1), 0600); err != nil {
+			return false, err
+		}
+	}
+
+	// Also store the new hashed output result.
+	if !outputMatches || h2 == "" { // If missing, we always write it out!
+		if err := os.WriteFile(obj.varDirPathOutput, []byte(sha256sum+"\n"), 0600); err != nil {
+			return false, err
+		}
+	}
+
+	fail = false // defer can exit safely!
+
+	return false, nil
+}
+
+// formatPrefix is a simple helper to add a format identifier for our hash.
+func (obj *DeployTar) formatPrefix() string {
+	return fmt.Sprintf("format:%d|%s", obj.Format, tar.Format(obj.Format))
+}
+
+// hashContent is a simple helper to run our hashing function.
+func (obj *DeployTar) hashContent(handle io.Reader) (string, error) {
+	hash := sha256.New()
+	if _, err := io.Copy(hash, handle); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(hash.Sum(nil)), nil
+}
+
+// hashFile is a helper that returns the hash of the specified file. If the file
+// doesn't exist, it returns the empty string. Otherwise it errors.
+func (obj *DeployTar) hashFile(file string) (string, error) {
+	f, err := os.Open(file) // io.Reader
+	if err != nil && !os.IsNotExist(err) {
+		// This is likely a permissions error.
+		return "", err
+
+	} else if err != nil {
+		return "", nil // File doesn't exist!
+	}
+
+	defer f.Close()
+
+	// File exists, lets hash it!
+
+	return obj.hashContent(f)
+}
+
+// hashFileAferoFs is a helper that returns the hash of the specified file with
+// an Afero fs. If the file doesn't exist, it returns the empty string.
+// Otherwise it errors.
+func (obj *DeployTar) hashFileAferoFs(fsys afero.Fs, file string) (string, error) {
+	f, err := fsys.Open(file) // io.Reader
+	if err != nil && !os.IsNotExist(err) {
+		// This is likely a permissions error.
+		return "", err
+
+	} else if err != nil {
+		return "", nil // File doesn't exist!
+	}
+
+	defer f.Close()
+
+	// File exists, lets hash it!
+
+	return obj.hashContent(f)
+}
+
+// readHashFile reads the hashed value that we stored for the output file.
+func (obj *DeployTar) readHashFile(file string, trim bool) (string, error) {
+	// TODO: Use io.ReadFull to avoid reading in a file that's too big!
+	if expected, err := os.ReadFile(file); err != nil && !os.IsNotExist(err) { // ([]byte, error)
+		// This is likely a permissions error?
+		return "", err
+
+	} else if err == nil {
+		if trim {
+			return strings.TrimSpace(string(expected)), nil
+		}
+		return string(expected), nil
+	}
+
+	// File doesn't exist!
+	return "", nil
+}
+
+// addFS is an edited copy of archive/tar's *Writer.AddFs function. This version
+// correctly adds the directories too! https://github.com/golang/go/issues/69459
+func (obj *DeployTar) addFS(tw *tar.Writer, fsys fs.FS) error {
+	return fs.WalkDir(fsys, ".", func(name string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if name == "." {
+			return nil
+		}
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+		// TODO: Handle symlinks when fs.ReadLinkFS is available. (#49580)
+		if !info.Mode().IsRegular() && !info.Mode().IsDir() {
+			return fmt.Errorf("deploy:tar: cannot add non-regular file")
+		}
+		h, err := tar.FileInfoHeader(info, "")
+		if err != nil {
+			return err
+		}
+		h.Name = name
+		h.Format = tar.Format(obj.Format)
+		if d.IsDir() {
+			h.Name += "/" // dir
+		}
+
+		if err := tw.WriteHeader(h); err != nil {
+			return err
+		}
+
+		if d.IsDir() {
+			return nil // no contents to copy in
+		}
+
+		f, err := fsys.Open(name)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		_, err = io.Copy(tw, f)
+		return err
+	})
+}
+
+// addAferoFs is an edited copy of archive/tar's *Writer.AddFs function but for
+// the deprecated Afero.Fs API. This version correctly adds the directories too!
+// https://github.com/golang/go/issues/69459
+func (obj *DeployTar) addAferoFs(tw *tar.Writer, fsys afero.Fs) error {
+	return afero.Walk(fsys, "/", func(name string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if name == "/" {
+			return nil
+		}
+		// TODO: Handle symlinks when fs.ReadLinkFS is available. (#49580)
+		if !info.Mode().IsRegular() && !info.Mode().IsDir() {
+			return fmt.Errorf("deploy:tar: cannot add non-regular file")
+		}
+		h, err := tar.FileInfoHeader(info, "")
+		if err != nil {
+			return err
+		}
+		h.Name = name
+		h.Format = tar.Format(obj.Format)
+		if info.IsDir() {
+			h.Name += "/" // dir
+		}
+
+		if err := tw.WriteHeader(h); err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil // no contents to copy in
+		}
+
+		f, err := fsys.Open(name)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		_, err = io.Copy(tw, f)
+		return err
+	})
+}
+
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *DeployTar) Cmp(r engine.Res) error {
+	// we can only compare DeployTar to others of the same resource kind
+	res, ok := r.(*DeployTar)
+	if !ok {
+		return fmt.Errorf("not a %s", obj.Kind())
+	}
+
+	if obj.Path != res.Path {
+		return fmt.Errorf("the Path differs")
+	}
+
+	if obj.Format != res.Format {
+		return fmt.Errorf("the Format differs")
+	}
+
+	return nil
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
+func (obj *DeployTar) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes DeployTar // indirection to avoid infinite recursion
+
+	def := obj.Default()        // get the default
+	res, ok := def.(*DeployTar) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to DeployTar")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = DeployTar(raw) // restore from indirection with type conversion!
+	return nil
+}

engine/resources/dhcp.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -894,6 +894,10 @@ func (obj *DHCPServerRes) handler4() func(net.PacketConn, net.Addr, *dhcpv4.DHCP
 			tmp.UpdateOption(dhcpv4.OptMessageType(dhcpv4.MessageTypeOffer))
 		case dhcpv4.MessageTypeRequest:
 			tmp.UpdateOption(dhcpv4.OptMessageType(dhcpv4.MessageTypeAck))
+		case dhcpv4.MessageTypeDecline:
+			// If mask is not set, some DHCP clients will DECLINE.
+			obj.init.Logf("handler4: Unhandled decline message: %+v", req)
+			return
 		default:
 			obj.init.Logf("handler4: Unhandled message type: %v", mt)
 			return
@@ -979,6 +983,7 @@ func (obj *DHCPServerRes) handler4() func(net.PacketConn, net.Addr, *dhcpv4.DHCP
 
 		if resp != nil {
 			if obj.init.Debug {
+				// NOTE: This is very useful for debugging!
 				obj.init.Logf("sending a DHCPv4 packet: %s", resp.Summary())
 			}
 			var peer net.Addr
@@ -1251,7 +1256,7 @@ func (obj *DHCPHostRes) handler4(data *HostData) (func(*dhcpv4.DHCPv4, *dhcpv4.D
 		// XXX: https://tools.ietf.org/html/rfc2132#section-3.3
 		// If both the subnet mask and the router option are specified
 		// in a DHCP reply, the subnet mask option MUST be first.
-		// XXX: Should we do this? Does it matter? Does the lib do it?
+		// If mask is not set, some DHCP clients will DECLINE.
 		resp.Options.Update(dhcpv4.OptSubnetMask(obj.ipv4Mask)) // net.IPMask
 
 		// nbp section
@@ -1714,7 +1719,7 @@ func (obj *DHCPRangeRes) Init(init *engine.Init) error {
 
 	obj.init.Logf("from: %s", obj.from)
 	obj.init.Logf("  to: %s", obj.to)
-	obj.init.Logf("mask: %s", obj.mask) // TODO: print as cidr or dotted quad
+	obj.init.Logf("mask: %s", netmaskAsQuadString(obj.mask))
 
 	return nil
 }
@@ -1932,8 +1937,8 @@ func (obj *DHCPRangeRes) handler4(data *HostData) (func(*dhcpv4.DHCPv4, *dhcpv4.
 		// XXX: https://tools.ietf.org/html/rfc2132#section-3.3
 		// If both the subnet mask and the router option are specified
 		// in a DHCP reply, the subnet mask option MUST be first.
-		// XXX: Should we do this? Does it matter? Does the lib do it?
-		//resp.Options.Update(dhcpv4.OptSubnetMask(obj.mask)) // net.IPMask
+		// If mask is not set, some DHCP clients will DECLINE.
+		resp.Options.Update(dhcpv4.OptSubnetMask(obj.mask)) // net.IPMask
 
 		// nbp section
 		if obj.opt66 != nil && req.IsOptionRequested(dhcpv4.OptionTFTPServerName) {
@@ -2049,3 +2054,9 @@ func checkValidNetmask(netmask net.IPMask) bool {
 	y := x + 1
 	return (y & x) == 0
 }
+
+// netmaskAsQuadString returns a dotted-quad string giving you something like:
+// 255.255.255.0 instead of ffffff00 which is what's seen when you print it now.
+func netmaskAsQuadString(netmask net.IPMask) string {
+	return fmt.Sprintf("%d.%d.%d.%d", netmask[0], netmask[1], netmask[2], netmask[3])
+}

engine/resources/doc.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/docker_container.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -47,6 +47,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/client"
 	"github.com/docker/go-connections/nat"
 )
@@ -234,7 +235,7 @@ func (obj *DockerContainerRes) CheckApply(ctx context.Context, apply bool) (bool
 	defer cancel()
 
 	// List any container whose name matches this resource.
-	opts := types.ContainerListOptions{
+	opts := container.ListOptions{
 		All:     true,
 		Filters: filters.NewArgs(filters.KeyValuePair{Key: "name", Value: obj.Name()}),
 	}
@@ -279,14 +280,14 @@ func (obj *DockerContainerRes) CheckApply(ctx context.Context, apply bool) (bool
 		if err := obj.containerStop(ctx, id, nil); err != nil {
 			return false, err
 		}
-		return false, obj.containerRemove(ctx, id, types.ContainerRemoveOptions{})
+		return false, obj.containerRemove(ctx, id, container.RemoveOptions{})
 	}
 
 	if destroy {
 		if err := obj.containerStop(ctx, id, nil); err != nil {
 			return false, err
 		}
-		if err := obj.containerRemove(ctx, id, types.ContainerRemoveOptions{}); err != nil {
+		if err := obj.containerRemove(ctx, id, container.RemoveOptions{}); err != nil {
 			return false, err
 		}
 		containerList = []types.Container{} // zero the list
@@ -294,7 +295,7 @@ func (obj *DockerContainerRes) CheckApply(ctx context.Context, apply bool) (bool
 
 	if len(containerList) == 0 { // no container was found
 		// Download the specified image if it doesn't exist locally.
-		p, err := obj.client.ImagePull(ctx, obj.Image, types.ImagePullOptions{})
+		p, err := obj.client.ImagePull(ctx, obj.Image, image.PullOptions{})
 		if err != nil {
 			return false, errwrap.Wrapf(err, "error pulling image")
 		}
@@ -334,11 +335,11 @@ func (obj *DockerContainerRes) CheckApply(ctx context.Context, apply bool) (bool
 		id = c.ID
 	}
 
-	return false, obj.containerStart(ctx, id, types.ContainerStartOptions{})
+	return false, obj.containerStart(ctx, id, container.StartOptions{})
 }
 
 // containerStart starts the specified container, and waits for it to start.
-func (obj *DockerContainerRes) containerStart(ctx context.Context, id string, opts types.ContainerStartOptions) error {
+func (obj *DockerContainerRes) containerStart(ctx context.Context, id string, opts container.StartOptions) error {
 	// Get an events channel for the container we're about to start.
 	eventOpts := types.EventsOptions{
 		Filters: filters.NewArgs(filters.KeyValuePair{Key: "container", Value: id}),
@@ -377,7 +378,7 @@ func (obj *DockerContainerRes) containerStop(ctx context.Context, id string, tim
 
 // containerRemove removes the specified container and waits for it to be
 // removed.
-func (obj *DockerContainerRes) containerRemove(ctx context.Context, id string, opts types.ContainerRemoveOptions) error {
+func (obj *DockerContainerRes) containerRemove(ctx context.Context, id string, opts container.RemoveOptions) error {
 	ch, errCh := obj.client.ContainerWait(ctx, id, container.WaitConditionRemoved)
 	obj.client.ContainerRemove(ctx, id, opts)
 	select {

engine/resources/docker_container_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -40,9 +40,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/image"
 )
 
 var res *DockerContainerRes
@@ -75,14 +75,14 @@ func BrokenTestContainerStart(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
-	if err := res.containerStart(ctx, id, types.ContainerStartOptions{}); err != nil {
+	if err := res.containerStart(ctx, id, container.StartOptions{}); err != nil {
 		t.Errorf("containerStart() error: %s", err)
 		return
 	}
 
 	l, err := res.client.ContainerList(
 		ctx,
-		types.ContainerListOptions{
+		container.ListOptions{
 			Filters: filters.NewArgs(
 				filters.KeyValuePair{Key: "id", Value: id},
 				filters.KeyValuePair{Key: "status", Value: "running"},
@@ -110,7 +110,7 @@ func BrokenTestContainerStop(t *testing.T) {
 
 	l, err := res.client.ContainerList(
 		ctx,
-		types.ContainerListOptions{
+		container.ListOptions{
 			Filters: filters.NewArgs(
 				filters.KeyValuePair{Key: "id", Value: id},
 			),
@@ -130,14 +130,14 @@ func BrokenTestContainerRemove(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
-	if err := res.containerRemove(ctx, id, types.ContainerRemoveOptions{}); err != nil {
+	if err := res.containerRemove(ctx, id, container.RemoveOptions{}); err != nil {
 		t.Errorf("containerRemove() error: %s", err)
 		return
 	}
 
 	l, err := res.client.ContainerList(
 		ctx,
-		types.ContainerListOptions{
+		container.ListOptions{
 			All: true,
 			Filters: filters.NewArgs(
 				filters.KeyValuePair{Key: "id", Value: id},
@@ -163,7 +163,7 @@ func setup() error {
 	res = &DockerContainerRes{}
 	res.Init(res.init)
 
-	p, err := res.client.ImagePull(ctx, "alpine", types.ImagePullOptions{})
+	p, err := res.client.ImagePull(ctx, "alpine", image.PullOptions{})
 	if err != nil {
 		return fmt.Errorf("error pulling image: %s", err)
 	}
@@ -195,7 +195,7 @@ func cleanup() error {
 
 	l, err := res.client.ContainerList(
 		ctx,
-		types.ContainerListOptions{
+		container.ListOptions{
 			All:     true,
 			Filters: filters.NewArgs(filters.KeyValuePair{Key: "id", Value: id}),
 		},
@@ -209,7 +209,7 @@ func cleanup() error {
 		if err := res.client.ContainerStop(ctx, id, stopOpts); err != nil {
 			return fmt.Errorf("error stopping container: %s", err)
 		}
-		if err := res.client.ContainerRemove(ctx, id, types.ContainerRemoveOptions{}); err != nil {
+		if err := res.client.ContainerRemove(ctx, id, container.RemoveOptions{}); err != nil {
 			return fmt.Errorf("error removing container: %s", err)
 		}
 	}

engine/resources/docker_image.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -44,6 +44,7 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/client"
 	errwrap "github.com/pkg/errors"
 )
@@ -188,7 +189,7 @@ func (obj *DockerImageRes) CheckApply(ctx context.Context, apply bool) (checkOK
 	ctx, cancel := context.WithTimeout(ctx, dockerImageCheckApplyCtxTimeout*time.Second)
 	defer cancel()
 
-	s, err := obj.client.ImageList(ctx, types.ImageListOptions{
+	s, err := obj.client.ImageList(ctx, image.ListOptions{
 		Filters: filters.NewArgs(filters.Arg("reference", obj.image)),
 	})
 	if err != nil {
@@ -211,14 +212,14 @@ func (obj *DockerImageRes) CheckApply(ctx context.Context, apply bool) (checkOK
 
 	if obj.State == "absent" {
 		// TODO: force? prune children?
-		if _, err := obj.client.ImageRemove(ctx, obj.image, types.ImageRemoveOptions{}); err != nil {
+		if _, err := obj.client.ImageRemove(ctx, obj.image, image.RemoveOptions{}); err != nil {
 			return false, errwrap.Wrapf(err, "error removing image")
 		}
 		return false, nil
 	}
 
 	// pull the image
-	p, err := obj.client.ImagePull(ctx, obj.image, types.ImagePullOptions{})
+	p, err := obj.client.ImagePull(ctx, obj.image, image.PullOptions{})
 	if err != nil {
 		return false, errwrap.Wrapf(err, "error pulling image")
 	}

engine/resources/exec.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -83,7 +83,10 @@ type ExecRes struct {
 
 	// Cwd is the dir to run the command in. If empty, then this will use
 	// the working directory of the calling process. (This process is mgmt,
-	// not the process being run here.)
+	// not the process being run here.) Keep in mind that if you're running
+	// this command as a user that does not have perms to the current
+	// directory, you may wish to set this to `/` to avoid hitting an error
+	// such as: `could not change directory to "/root": Permission denied`.
 	Cwd string `lang:"cwd" yaml:"cwd"`
 
 	// Shell is the (optional) shell to use to run the cmd. If you specify

engine/resources/exec_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/file.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -40,6 +40,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -238,18 +239,19 @@ func (obj *FileRes) isDir() bool {
 // the case where the mode is not specified. The caller should check obj.Mode is
 // not empty.
 func (obj *FileRes) mode() (os.FileMode, error) {
+	// First check if this is an octal number.
 	if n, err := strconv.ParseInt(obj.Mode, 8, 32); err == nil {
 		return os.FileMode(n), nil
 	}
 
 	// Try parsing symbolically by first getting the files current mode.
-	stat, err := os.Stat(obj.getPath())
-	if err != nil {
-		return os.FileMode(0), errwrap.Wrapf(err, "failed to get the current file mode")
+	from := os.FileMode(0) // default
+	if stat, err := os.Stat(obj.getPath()); err == nil {
+		from = stat.Mode()
 	}
 
 	modes := strings.Split(obj.Mode, ",")
-	m, err := engineUtil.ParseSymbolicModes(modes, stat.Mode(), FileModeAllowAssign)
+	m, err := engineUtil.ParseSymbolicModes(modes, from, FileModeAllowAssign)
 	if err != nil {
 		return os.FileMode(0), errwrap.Wrapf(err, "mode should be an octal number or symbolic mode (%s)", obj.Mode)
 	}
@@ -352,13 +354,6 @@ func (obj *FileRes) Validate() error {
 			return fmt.Errorf("can't set Owner or Group on this platform")
 		}
 	}
-	if _, err := engineUtil.GetUID(obj.Owner); obj.Owner != "" && err != nil {
-		return err
-	}
-
-	if _, err := engineUtil.GetGID(obj.Group); obj.Group != "" && err != nil {
-		return err
-	}
 
 	// TODO: should we silently ignore this error or include it?
 	//if obj.State == FileStateAbsent && obj.Mode != "" {
@@ -555,11 +550,16 @@ func (obj *FileRes) fileCheckApply(ctx context.Context, apply bool, src io.ReadS
 		obj.init.Logf("fileCheckApply: %v -> %s", src, dst)
 	}
 
+	length := int64(-1)
+
 	srcFile, isFile := src.(*os.File)
-	_, isBytes := src.(*bytes.Reader) // supports seeking!
+	srcReader, isBytes := src.(*bytes.Reader) // supports seeking!
 	if !isFile && !isBytes {
 		return "", false, fmt.Errorf("can't open src as either file or buffer")
 	}
+	if isBytes {
+		length = int64(srcReader.Len())
+	}
 
 	var srcStat os.FileInfo
 	if isFile {
@@ -572,6 +572,8 @@ func (obj *FileRes) fileCheckApply(ctx context.Context, apply bool, src io.ReadS
 		if !srcStat.Mode().IsRegular() { // can't copy non-regular files or dirs
 			return "", false, fmt.Errorf("non-regular src file: %s (%q)", srcStat.Name(), srcStat.Mode())
 		}
+
+		length = srcStat.Size()
 	}
 
 	dstFile, err := os.Open(dst)
@@ -610,7 +612,7 @@ func (obj *FileRes) fileCheckApply(ctx context.Context, apply bool, src io.ReadS
 		}
 		// FIXME: respect obj.Recurse here...
 		// there is a dir here, where we want a file...
-		obj.init.Logf("fileCheckApply: removing (force): %s", cleanDst)
+		obj.init.Logf("removing (force): %s", cleanDst)
 		if err := os.RemoveAll(cleanDst); err != nil { // dangerous ;)
 			return "", false, err
 		}
@@ -656,7 +658,7 @@ func (obj *FileRes) fileCheckApply(ctx context.Context, apply bool, src io.ReadS
 		return sha256sum, false, nil
 	}
 	if obj.init.Debug {
-		obj.init.Logf("fileCheckApply: apply: %v -> %s", src, dst)
+		obj.init.Logf("apply: %v -> %s", src, dst)
 	}
 
 	dstClose() // unlock file usage so we can write to it
@@ -676,13 +678,16 @@ func (obj *FileRes) fileCheckApply(ctx context.Context, apply bool, src io.ReadS
 	// syscall.Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error)
 
 	// TODO: should we offer a way to cancel the copy on ^C ?
-	if obj.init.Debug {
-		obj.init.Logf("fileCheckApply: copy: %v -> %s", src, dst)
+	if isFile {
+		obj.init.Logf("copy %d bytes from: %v", length, src)
+	} else if isBytes {
+		obj.init.Logf("copy %d bytes", length)
 	}
+
 	if n, err := io.Copy(dstFile, src); err != nil {
 		return sha256sum, false, err
 	} else if obj.init.Debug {
-		obj.init.Logf("fileCheckApply: copied: %v", n)
+		obj.init.Logf("copied: %v", n)
 	}
 	return sha256sum, false, dstFile.Sync()
 }
@@ -709,7 +714,7 @@ func (obj *FileRes) dirCheckApply(ctx context.Context, apply bool) (bool, error)
 	// the path exists and is not a directory
 	// delete the file if force is given
 	if err == nil && !fileInfo.IsDir() {
-		obj.init.Logf("dirCheckApply: removing (force): %s", obj.getPath())
+		obj.init.Logf("removing (force): %s", obj.getPath())
 		if err := os.Remove(obj.getPath()); err != nil {
 			return false, err
 		}
@@ -725,6 +730,7 @@ func (obj *FileRes) dirCheckApply(ctx context.Context, apply bool) (bool, error)
 
 	if obj.Recurse {
 		// TODO: add recurse limit here
+		obj.init.Logf("mkdir -p -m %s", mode)
 		return false, os.MkdirAll(obj.getPath(), mode)
 	}
 
@@ -738,7 +744,7 @@ func (obj *FileRes) dirCheckApply(ctx context.Context, apply bool) (bool, error)
 // with the exception that a sync *can* convert a file to a dir, or vice-versa.
 func (obj *FileRes) syncCheckApply(ctx context.Context, apply bool, src, dst string, excludes []string) (bool, error) {
 	if obj.init.Debug {
-		obj.init.Logf("syncCheckApply: %s -> %s", src, dst)
+		obj.init.Logf("sync: %s -> %s", src, dst)
 	}
 	// an src of "" is now supported, if dst is a dir
 	if dst == "" {
@@ -760,12 +766,12 @@ func (obj *FileRes) syncCheckApply(ctx context.Context, apply bool, src, dst str
 
 	if !srcIsDir && !dstIsDir && src != "" {
 		if obj.init.Debug {
-			obj.init.Logf("syncCheckApply: %s -> %s", src, dst)
+			obj.init.Logf("sync: %s -> %s", src, dst)
 		}
 		fin, err := os.Open(src)
 		if err != nil {
 			if obj.init.Debug && os.IsNotExist(err) { // if we get passed an empty src
-				obj.init.Logf("syncCheckApply: missing src: %s", src)
+				obj.init.Logf("missing src: %s", src)
 			}
 			return false, err
 		}
@@ -787,7 +793,9 @@ func (obj *FileRes) syncCheckApply(ctx context.Context, apply bool, src, dst str
 			return false, err
 		}
 		smartSrc = mapPaths(srcFiles)
-		obj.init.Logf("syncCheckApply: srcFiles: %v", srcFiles)
+		if obj.init.Debug {
+			obj.init.Logf("srcFiles: %v", printFiles(smartSrc))
+		}
 	}
 
 	dstFiles, err := ReadDir(dst)
@@ -795,7 +803,9 @@ func (obj *FileRes) syncCheckApply(ctx context.Context, apply bool, src, dst str
 		return false, err
 	}
 	smartDst := mapPaths(dstFiles)
-	obj.init.Logf("syncCheckApply: dstFiles: %v", dstFiles)
+	if obj.init.Debug {
+		obj.init.Logf("dstFiles: %v", printFiles(smartDst))
+	}
 
 	for relPath, fileInfo := range smartSrc {
 		absSrc := fileInfo.AbsPath // absolute path
@@ -819,7 +829,7 @@ func (obj *FileRes) syncCheckApply(ctx context.Context, apply bool, src, dst str
 					if absCleanDst == "" || absCleanDst == "/" {
 						return false, fmt.Errorf("don't want to remove root") // safety
 					}
-					obj.init.Logf("syncCheckApply: removing (force): %s", absCleanDst)
+					obj.init.Logf("removing (force): %s", absCleanDst)
 					if err := os.Remove(absCleanDst); err != nil {
 						return false, err
 					}
@@ -827,7 +837,7 @@ func (obj *FileRes) syncCheckApply(ctx context.Context, apply bool, src, dst str
 				}
 
 				if obj.init.Debug {
-					obj.init.Logf("syncCheckApply: mkdir -m %s '%s'", fileInfo.Mode(), absDst)
+					obj.init.Logf("mkdir -m %s '%s'", fileInfo.Mode(), absDst)
 				}
 				if err := os.Mkdir(absDst, fileInfo.Mode()); err != nil {
 					return false, err
@@ -838,11 +848,11 @@ func (obj *FileRes) syncCheckApply(ctx context.Context, apply bool, src, dst str
 		}
 
 		if obj.init.Debug {
-			obj.init.Logf("syncCheckApply: recurse: %s -> %s", absSrc, absDst)
+			obj.init.Logf("recurse: %s -> %s", absSrc, absDst)
 		}
 		if obj.Recurse {
 			if c, err := obj.syncCheckApply(ctx, apply, absSrc, absDst, excludes); err != nil { // recurse
-				return false, errwrap.Wrapf(err, "syncCheckApply: recurse failed")
+				return false, errwrap.Wrapf(err, "recurse failed")
 			} else if !c { // don't let subsequent passes make this true
 				checkOK = false
 			}
@@ -887,7 +897,7 @@ func (obj *FileRes) syncCheckApply(ctx context.Context, apply bool, src, dst str
 			if isExcluded(absDst) { // skip removing excluded files
 				continue
 			}
-			obj.init.Logf("syncCheckApply: removing: %s", absCleanDst)
+			obj.init.Logf("removing: %s", absCleanDst)
 			if apply {
 				if err := os.RemoveAll(absCleanDst); err != nil { // dangerous ;)
 					return false, err
@@ -897,16 +907,16 @@ func (obj *FileRes) syncCheckApply(ctx context.Context, apply bool, src, dst str
 			continue
 		}
 		_ = absSrc
-		//obj.init.Logf("syncCheckApply: recurse rm: %s -> %s", absSrc, absDst)
+		//obj.init.Logf("recurse rm: %s -> %s", absSrc, absDst)
 		//if c, err := obj.syncCheckApply(ctx, apply, absSrc, absDst, excludes); err != nil {
-		//	return false, errwrap.Wrapf(err, "syncCheckApply: recurse rm failed")
+		//	return false, errwrap.Wrapf(err, "recurse rm failed")
 		//} else if !c { // don't let subsequent passes make this true
 		//	checkOK = false
 		//}
 		//if isExcluded(absDst) { // skip removing excluded files
 		//	continue
 		//}
-		//obj.init.Logf("syncCheckApply: removing: %s", absCleanDst)
+		//obj.init.Logf("removing: %s", absCleanDst)
 		//if apply { // safety
 		//	if err := os.Remove(absCleanDst); err != nil {
 		//		return false, err
@@ -953,7 +963,7 @@ func (obj *FileRes) stateCheckApply(ctx context.Context, apply bool) (bool, erro
 		if p == "/" {
 			return false, fmt.Errorf("don't want to remove root") // safety
 		}
-		obj.init.Logf("stateCheckApply: removing: %s", p)
+		obj.init.Logf("removing: %s", p)
 		// TODO: add recurse limit here
 		if obj.Recurse {
 			return false, os.RemoveAll(p) // dangerous ;)
@@ -1058,13 +1068,13 @@ func (obj *FileRes) sourceCheckApply(ctx context.Context, apply bool) (bool, err
 		}
 	}
 	if obj.init.Debug {
-		obj.init.Logf("syncCheckApply: excludes: %+v", excludes)
+		obj.init.Logf("excludes: %+v", excludes)
 	}
 
 	// XXX: should this work with obj.Purge && obj.Source != "" or not?
 	checkOK, err := obj.syncCheckApply(ctx, apply, obj.Source, obj.getPath(), excludes)
 	if err != nil {
-		obj.init.Logf("syncCheckApply: error: %v", err)
+		obj.init.Logf("error: %v", err)
 		return false, err
 	}
 
@@ -1192,6 +1202,7 @@ func (obj *FileRes) chownCheckApply(ctx context.Context, apply bool) (bool, erro
 		return false, nil
 	}
 
+	obj.init.Logf("chown %s:%s", obj.Owner, obj.Group)
 	return false, os.Chown(obj.getPath(), expectedUID, expectedGID)
 }
 
@@ -1226,6 +1237,7 @@ func (obj *FileRes) chmodCheckApply(ctx context.Context, apply bool) (bool, erro
 		return false, nil
 	}
 
+	obj.init.Logf("chmod %s", obj.Mode)
 	return false, os.Chmod(obj.getPath(), mode)
 }
 
@@ -1724,3 +1736,20 @@ func mapPaths(fileInfos []FileInfo) map[string]FileInfo {
 	}
 	return paths
 }
+
+// printFiles is a pretty print function to make log messages less ugly.
+func printFiles(fileInfos map[string]FileInfo) string {
+	s := ""
+	keys := []string{}
+	for k := range fileInfos {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for i, k := range keys {
+		s += fileInfos[k].RelPath
+		if i < len(keys)-1 {
+			s += ", "
+		}
+	}
+	return s
+}

engine/resources/file_test.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/firewalld.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -318,7 +318,9 @@ func (obj *FirewalldRes) CheckApply(ctx context.Context, apply bool) (bool, erro
 		if obj.zone == "" {
 			return false, fmt.Errorf("unexpected empty zone")
 		}
-		obj.init.Logf("zone: %s\n", obj.zone)
+		if obj.init.Debug {
+			obj.init.Logf("zone: %s", obj.zone)
+		}
 	}
 
 	checkOK := true

engine/resources/group.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -135,8 +135,6 @@ func (obj *GroupRes) Watch(ctx context.Context) error {
 
 // CheckApply method for Group resource.
 func (obj *GroupRes) CheckApply(ctx context.Context, apply bool) (bool, error) {
-	obj.init.Logf("CheckApply(%t)", apply)
-
 	// check if the group exists
 	exists := true
 	group, err := user.LookupGroup(obj.Name())

engine/resources/gzip.go

@@ -0,0 +1,518 @@
+// Mgmt
+// Copyright (C) James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package resources
+
+import (
+	"compress/gzip"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/engine/traits"
+	"github.com/purpleidea/mgmt/lang/funcs/vars"
+	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/lang/types"
+	"github.com/purpleidea/mgmt/util/errwrap"
+	"github.com/purpleidea/mgmt/util/recwatch"
+)
+
+func init() {
+	engine.RegisterResource("gzip", func() engine.Res { return &GzipRes{} })
+
+	// const.res.gzip.level.no_compression = 0
+	// const.res.gzip.level.best_speed = 1
+	// const.res.gzip.level.best_compression = 9
+	// const.res.gzip.level.default_compression = -1
+	// const.res.gzip.level.huffman_only = -2
+	vars.RegisterResourceParams("gzip", map[string]map[string]func() interfaces.Var{
+		"level": {
+			"no_compression": func() interfaces.Var {
+				return &types.IntValue{
+					V: gzip.NoCompression,
+				}
+			},
+			"best_speed": func() interfaces.Var {
+				return &types.IntValue{
+					V: gzip.BestSpeed,
+				}
+			},
+			"best_compression": func() interfaces.Var {
+				return &types.IntValue{
+					V: gzip.BestCompression,
+				}
+			},
+			"default_compression": func() interfaces.Var {
+				return &types.IntValue{
+					V: gzip.DefaultCompression,
+				}
+			},
+			"huffman_only": func() interfaces.Var {
+				return &types.IntValue{
+					V: gzip.HuffmanOnly,
+				}
+			},
+		},
+	})
+}
+
+// GzipRes is a resource that compresses a path or some raw data using gzip. The
+// name of the resource is the path to the resultant compressed file. The input
+// can either come from a file path if specified with Input or it looks at the
+// Content field for raw data. It uses hashes to determine if something was
+// changed, so as a result, this may not be suitable if you can create a sha256
+// hash collision.
+// TODO: support send/recv to send the output instead of writing to a file?
+type GzipRes struct {
+	traits.Base // add the base methods without re-implementation
+
+	init *engine.Init
+
+	// Path, which defaults to the name if not specified, represents the
+	// destination path for the compressed file being created. It must be an
+	// absolute path, and as a result must start with a slash. Since it is a
+	// file, it must not end with a slash.
+	Path string `lang:"path" yaml:"path"`
+
+	// Input represents the input file to be compressed. It must be an
+	// absolute path, and as a result must start with a slash. Since it is a
+	// file, it must not end with a slash. If this is specified, we use it,
+	// otherwise we use the Content parameter.
+	Input *string `lang:"input" yaml:"input"`
+
+	// Content is the raw data to compress. If Input is not specified, then
+	// we use this parameter. If you forget to specify both of these, then
+	// you will compress zero-length data!
+	// TODO: If this is also empty should we just error at Validate?
+	// FIXME: Do we need []byte here? Do we need a binary type?
+	Content string `lang:"content" yaml:"content"`
+
+	// Level is the compression level to use. If you change this, then the
+	// file will get recompressed. The available values are:
+	// const.res.gzip.level.no_compression, const.res.gzip.level.best_speed,
+	// const.res.gzip.level.best_compression,
+	// const.res.gzip.level.default_compression, and
+	// const.res.gzip.level.huffman_only.
+	Level int `lang:"level" yaml:"level"`
+
+	// SendOnly specifies that we don't write the file to disk, and as a
+	// result, the output is only be accessible by the send/recv mechanism.
+	// TODO: Rename this?
+	// TODO: Not implemented
+	//SendOnly bool `lang:"sendonly" yaml:"sendonly"`
+
+	// sha256sum is the hash of the content if it's using obj.Content here.
+	sha256sum string
+
+	// varDirPathInput is the path we use to store the content hash.
+	varDirPathInput string
+
+	// varDirPathOutput is the path we use to store the output file hash.
+	varDirPathOutput string
+}
+
+// getPath returns the actual path to use for this resource. It computes this
+// after analysis of the Path and Name.
+func (obj *GzipRes) getPath() string {
+	p := obj.Path
+	if obj.Path == "" { // use the name as the path default if missing
+		p = obj.Name()
+	}
+	return p
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *GzipRes) Default() engine.Res {
+	return &GzipRes{
+		Level: gzip.DefaultCompression,
+	}
+}
+
+// Validate if the params passed in are valid data.
+func (obj *GzipRes) Validate() error {
+	if obj.getPath() == "" {
+		return fmt.Errorf("path is empty")
+	}
+	if !strings.HasPrefix(obj.getPath(), "/") {
+		return fmt.Errorf("path must be absolute")
+	}
+	if strings.HasSuffix(obj.getPath(), "/") {
+		return fmt.Errorf("path must not end with a slash")
+	}
+
+	if obj.Input != nil {
+		if !strings.HasPrefix(*obj.Input, "/") {
+			return fmt.Errorf("input must be absolute")
+		}
+		if strings.HasSuffix(*obj.Input, "/") {
+			return fmt.Errorf("input must not end with a slash")
+		}
+	}
+
+	// This validation logic was observed in the gzip source code.
+	if obj.Level < gzip.HuffmanOnly || obj.Level > gzip.BestCompression {
+		return fmt.Errorf("invalid compression level: %d", obj.Level)
+	}
+
+	return nil
+}
+
+// Init runs some startup code for this resource.
+func (obj *GzipRes) Init(init *engine.Init) error {
+	obj.init = init // save for later
+
+	dir, err := obj.init.VarDir("")
+	if err != nil {
+		return errwrap.Wrapf(err, "could not get VarDir in Init()")
+	}
+	// return unique files
+	obj.varDirPathInput = path.Join(dir, "input.sha256")
+	obj.varDirPathOutput = path.Join(dir, "output.sha256")
+
+	if obj.Input != nil {
+		return nil
+	}
+
+	// This is all stuff that's done when we're using obj.Content instead...
+	sha256sum, err := obj.hashContent(strings.NewReader(obj.Content))
+	if err != nil {
+		return err
+	}
+	obj.sha256sum = sha256sum
+
+	return nil
+}
+
+// Cleanup is run by the engine to clean up after the resource is done.
+func (obj *GzipRes) Cleanup() error {
+	return nil
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *GzipRes) Watch(ctx context.Context) error {
+	recurse := false // single file
+
+	recWatcher, err := recwatch.NewRecWatcher(obj.getPath(), recurse)
+	if err != nil {
+		return err
+	}
+	defer recWatcher.Close()
+
+	var events chan recwatch.Event
+
+	if obj.Input != nil {
+		recWatcher, err := recwatch.NewRecWatcher(*obj.Input, recurse)
+		if err != nil {
+			return err
+		}
+		defer recWatcher.Close()
+		events = recWatcher.Events()
+	}
+
+	obj.init.Running() // when started, notify engine that we're running
+
+	var send = false // send event?
+	for {
+		select {
+		case event, ok := <-recWatcher.Events():
+			if !ok { // channel shutdown
+				// TODO: Should this be an error? Previously it
+				// was a `return nil`, and i'm not sure why...
+				//return nil
+				return fmt.Errorf("unexpected close")
+			}
+			if err := event.Error; err != nil {
+				return errwrap.Wrapf(err, "unknown %s watcher error", obj)
+			}
+			if obj.init.Debug { // don't access event.Body if event.Error isn't nil
+				obj.init.Logf("event(%s): %v", event.Body.Name, event.Body.Op)
+			}
+			send = true
+
+		case event, ok := <-events:
+			if !ok { // channel shutdown
+				return fmt.Errorf("unexpected close")
+			}
+			if err := event.Error; err != nil {
+				return err
+			}
+			if obj.init.Debug { // don't access event.Body if event.Error isn't nil
+				obj.init.Logf("event(%s): %v", event.Body.Name, event.Body.Op)
+			}
+			send = true
+
+		case <-ctx.Done(): // closed by the engine to signal shutdown
+			return nil
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			obj.init.Event() // notify engine of an event (this can block)
+		}
+	}
+}
+
+// CheckApply checks the resource state and applies the resource if the bool
+// input is true. It returns error info and if the state check passed or not.
+// This is where we actually do the compression work when needed.
+func (obj *GzipRes) CheckApply(ctx context.Context, apply bool) (bool, error) {
+
+	h1, err := obj.hashFile(obj.getPath()) // output
+	if err != nil {
+		return false, err
+	}
+
+	h2, err := obj.readHashFile(obj.varDirPathOutput)
+	if err != nil {
+		return false, err
+	}
+
+	i1 := obj.sha256sum
+	if obj.Input != nil {
+		h, err := obj.hashFile(*obj.Input)
+		if err != nil {
+			return false, err
+		}
+		i1 = h
+	}
+	i1 = obj.levelPrefix() + i1 // add the level prefix so it is considered
+
+	i2, err := obj.readHashFile(obj.varDirPathInput)
+	if err != nil {
+		return false, err
+	}
+
+	// We're cheating by computing this before we know if we errored!
+	inputMatches := i1 == i2
+	outputMatches := h1 == h2
+	if err == nil && inputMatches && outputMatches {
+		// If the two hashes match, we assume that the file is correct!
+		// The file has to also exist of course...
+		return true, nil
+	}
+
+	if !apply {
+		return false, nil
+	}
+
+	fail := true // assume we have a failure
+
+	defer func() {
+		if !fail {
+			return
+		}
+		// Don't leave a partial file lying around...
+		obj.init.Logf("removing partial gzip file")
+		err := os.Remove(obj.getPath())
+		if err == nil || os.IsNotExist(err) {
+			return
+		}
+		obj.init.Logf("error removing corrupt gzip file: %v", err)
+	}()
+
+	// FIXME: Do we instead want to write to a tmp file and do a move once
+	// we finish writing to be atomic here and avoid partial corrupt files?
+	// FIXME: Add a param called Atomic to specify that behaviour. It's
+	// instant so that might be preferred as it might generate fewer events,
+	// but there's a chance it's copying from obj.init.VarDir() to a
+	// different filesystem.
+	outputFile, err := os.Create(obj.getPath()) // io.Writer
+	if err != nil {
+		return false, err
+	}
+	//defer outputFile.Sync() // not needed?
+	defer outputFile.Close()
+
+	hash := sha256.New()
+
+	// Write to both to avoid needing to wait for fsync to calculate hash!
+	multiWriter := io.MultiWriter(outputFile, hash)
+
+	gzipWriter, err := gzip.NewWriterLevel(multiWriter, obj.Level) // (*gzip.Writer, error)
+	if err != nil {
+		return false, err
+	}
+
+	var input io.Reader
+	if obj.Input != nil {
+		f, err := os.Open(*obj.Input) // io.Reader
+		if err != nil && !os.IsNotExist(err) {
+			// This is likely a permissions error.
+			return false, err
+
+		} else if err != nil {
+			return false, err // File doesn't exist!
+		}
+		defer f.Close()
+		input = f
+
+	} else {
+		input = strings.NewReader(obj.Content)
+	}
+
+	// Copy the input file into the writer, which writes it out compressed.
+	count, err := io.Copy(gzipWriter, input) // dst, src
+	if err != nil {
+		gzipWriter.Close() // Might as well always close!
+		return false, err
+	}
+
+	// NOTE: Must run this before hashing so that it includes the footer!
+	if err := gzipWriter.Close(); err != nil {
+		return false, err
+	}
+	sha256sum := hex.EncodeToString(hash.Sum(nil))
+
+	obj.init.Logf("wrote %d gzipped bytes", count)
+
+	// After gzip is successfully written, store the hashed input result.
+	if !inputMatches {
+		if err := os.WriteFile(obj.varDirPathInput, []byte(i1+"\n"), 0600); err != nil {
+			return false, err
+		}
+	}
+
+	// Also store the new hashed output result.
+	if !outputMatches || h2 == "" { // If missing, we always write it out!
+		if err := os.WriteFile(obj.varDirPathOutput, []byte(sha256sum+"\n"), 0600); err != nil {
+			return false, err
+		}
+	}
+
+	fail = false // defer can exit safely!
+
+	return false, nil
+}
+
+// levelPrefix is a simple helper to add a level identifier for our hash.
+func (obj *GzipRes) levelPrefix() string {
+	return fmt.Sprintf("level:%d|", obj.Level)
+}
+
+// hashContent is a simple helper to run our hashing function.
+func (obj *GzipRes) hashContent(handle io.Reader) (string, error) {
+	hash := sha256.New()
+	if _, err := io.Copy(hash, handle); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(hash.Sum(nil)), nil
+}
+
+// hashFile is a helper that returns the hash of the specified file. If the file
+// doesn't exist, it returns the empty string. Otherwise it errors.
+func (obj *GzipRes) hashFile(file string) (string, error) {
+	f, err := os.Open(file) // io.Reader
+	if err != nil && !os.IsNotExist(err) {
+		// This is likely a permissions error.
+		return "", err
+
+	} else if err != nil {
+		return "", nil // File doesn't exist!
+	}
+
+	defer f.Close()
+
+	// File exists, lets hash it!
+
+	return obj.hashContent(f)
+}
+
+// readHashFile reads the hashed value that we stored for the output file.
+func (obj *GzipRes) readHashFile(file string) (string, error) {
+	// TODO: Use io.ReadFull to avoid reading in a file that's too big!
+	if expected, err := os.ReadFile(file); err != nil && !os.IsNotExist(err) { // ([]byte, error)
+		// This is likely a permissions error?
+		return "", err
+
+	} else if err == nil {
+		return strings.TrimSpace(string(expected)), nil
+	}
+
+	// File doesn't exist!
+	return "", nil
+}
+
+// Cmp compares two resources and returns an error if they are not equivalent.
+func (obj *GzipRes) Cmp(r engine.Res) error {
+	// we can only compare GzipRes to others of the same resource kind
+	res, ok := r.(*GzipRes)
+	if !ok {
+		return fmt.Errorf("not a %s", obj.Kind())
+	}
+
+	if obj.Path != res.Path {
+		return fmt.Errorf("the Path differs")
+	}
+
+	if (obj.Input == nil) != (res.Input == nil) { // xor
+		return fmt.Errorf("the Input differs")
+	}
+	if obj.Input != nil && res.Input != nil {
+		if *obj.Input != *res.Input { // compare the strings
+			return fmt.Errorf("the contents of Input differ")
+		}
+	}
+
+	if obj.Content != res.Content {
+		return fmt.Errorf("the Content differs")
+	}
+
+	if obj.Level != res.Level {
+		return fmt.Errorf("the Level differs")
+	}
+
+	return nil
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct. It is
+// primarily useful for setting the defaults.
+func (obj *GzipRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes GzipRes // indirection to avoid infinite recursion
+
+	def := obj.Default()      // get the default
+	res, ok := def.(*GzipRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to GzipRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = GzipRes(raw) // restore from indirection with type conversion!
+	return nil
+}

engine/resources/hetzner_vm.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/hostname.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -39,6 +39,7 @@ import (
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/util/errwrap"
+	"github.com/purpleidea/mgmt/util/recwatch"
 
 	"github.com/godbus/dbus/v5"
 )
@@ -57,7 +58,10 @@ const (
 // resource is insufficient for the resource to do any useful work.
 var ErrResourceInsufficientParameters = errors.New("insufficient parameters for this resource")
 
-// HostnameRes is a resource that allows setting and watching the hostname.
+// HostnameRes is a resource that allows setting and watching the hostname. If
+// you don't specify any parameters, the Name is used. The Hostname field is
+// used if none of the other parameters are used. If the parameters are set to
+// the empty string, then those variants are not managed by the resource.
 type HostnameRes struct {
 	traits.Base // add the base methods without re-implementation
 
@@ -72,22 +76,54 @@ type HostnameRes struct {
 
 	// PrettyHostname is a free-form UTF8 host name for presentation to the
 	// user.
-	PrettyHostname string `lang:"pretty_hostname" yaml:"pretty_hostname"`
+	PrettyHostname *string `lang:"pretty_hostname" yaml:"pretty_hostname"`
 
 	// StaticHostname is the one configured in /etc/hostname or a similar
 	// file. It is chosen by the local user. It is not always in sync with
 	// the current host name as returned by the gethostname() system call.
-	StaticHostname string `lang:"static_hostname" yaml:"static_hostname"`
+	StaticHostname *string `lang:"static_hostname" yaml:"static_hostname"`
 
 	// TransientHostname is the one configured via the kernel's
 	// sethostbyname(). It can be different from the static hostname in case
 	// DHCP or mDNS have been configured to change the name based on network
 	// information.
-	TransientHostname string `lang:"transient_hostname" yaml:"transient_hostname"`
+	TransientHostname *string `lang:"transient_hostname" yaml:"transient_hostname"`
 
 	conn *dbus.Conn
 }
 
+func (obj *HostnameRes) getHostname() string {
+	if obj.Hostname != "" {
+		return obj.Hostname
+	}
+
+	return obj.Name()
+}
+
+func (obj *HostnameRes) getPrettyHostname() string {
+	if obj.PrettyHostname != nil {
+		return *obj.PrettyHostname // this may be empty!
+	}
+
+	return obj.getHostname()
+}
+
+func (obj *HostnameRes) getStaticHostname() string {
+	if obj.StaticHostname != nil {
+		return *obj.StaticHostname // this may be empty!
+	}
+
+	return obj.getHostname()
+}
+
+func (obj *HostnameRes) getTransientHostname() string {
+	if obj.TransientHostname != nil {
+		return *obj.TransientHostname // this may be empty!
+	}
+
+	return obj.getHostname()
+}
+
 // Default returns some sensible defaults for this resource.
 func (obj *HostnameRes) Default() engine.Res {
 	return &HostnameRes{}
@@ -95,7 +131,10 @@ func (obj *HostnameRes) Default() engine.Res {
 
 // Validate if the params passed in are valid data.
 func (obj *HostnameRes) Validate() error {
-	if obj.PrettyHostname == "" && obj.StaticHostname == "" && obj.TransientHostname == "" {
+	a := obj.getPrettyHostname() == ""
+	b := obj.getStaticHostname() == ""
+	c := obj.getTransientHostname() == ""
+	if a && b && c && obj.getHostname() == "" {
 		return ErrResourceInsufficientParameters
 	}
 	return nil
@@ -105,15 +144,6 @@ func (obj *HostnameRes) Validate() error {
 func (obj *HostnameRes) Init(init *engine.Init) error {
 	obj.init = init // save for later
 
-	if obj.PrettyHostname == "" {
-		obj.PrettyHostname = obj.Hostname
-	}
-	if obj.StaticHostname == "" {
-		obj.StaticHostname = obj.Hostname
-	}
-	if obj.TransientHostname == "" {
-		obj.TransientHostname = obj.Hostname
-	}
 	return nil
 }
 
@@ -124,6 +154,13 @@ func (obj *HostnameRes) Cleanup() error {
 
 // Watch is the primary listener for this resource and it outputs events.
 func (obj *HostnameRes) Watch(ctx context.Context) error {
+	recurse := false // single file
+	recWatcher, err := recwatch.NewRecWatcher("/etc/hostname", recurse)
+	if err != nil {
+		return err
+	}
+	defer recWatcher.Close()
+
 	// if we share the bus with others, we will get each others messages!!
 	bus, err := util.SystemBusPrivateUsable() // don't share the bus connection!
 	if err != nil {
@@ -149,7 +186,23 @@ func (obj *HostnameRes) Watch(ctx context.Context) error {
 	var send = false // send event?
 	for {
 		select {
-		case <-signals:
+		case _, ok := <-signals:
+			if !ok { // channel shutdown
+				return fmt.Errorf("unexpected close")
+			}
+			//signals = nil
+			send = true
+
+		case event, ok := <-recWatcher.Events():
+			if !ok { // channel shutdown
+				return fmt.Errorf("unexpected close")
+			}
+			if err := event.Error; err != nil {
+				return err
+			}
+			if obj.init.Debug { // don't access event.Body if event.Error isn't nil
+				obj.init.Logf("event(%s): %v", event.Body.Name, event.Body.Op)
+			}
 			send = true
 
 		case <-ctx.Done(): // closed by the engine to signal shutdown
@@ -189,10 +242,10 @@ func (obj *HostnameRes) updateHostnameProperty(object dbus.BusObject, expectedVa
 	}
 
 	// attempting to apply the changes
-	obj.init.Logf("Changing %s: %s => %s", property, propertyValue, expectedValue)
 	if err := object.Call("org.freedesktop.hostname1."+setterName, 0, expectedValue, false).Err; err != nil {
 		return false, errwrap.Wrapf(err, "failed to call org.freedesktop.hostname1.%s", setterName)
 	}
+	obj.init.Logf("changed %s: `%s` => `%s`", property, propertyValue, expectedValue)
 
 	// all good changes should now be applied again
 	return false, nil
@@ -209,22 +262,22 @@ func (obj *HostnameRes) CheckApply(ctx context.Context, apply bool) (bool, error
 	hostnameObject := conn.Object(hostname1Iface, hostname1Path)
 
 	checkOK := true
-	if obj.PrettyHostname != "" {
-		propertyCheckOK, err := obj.updateHostnameProperty(hostnameObject, obj.PrettyHostname, "PrettyHostname", "SetPrettyHostname", apply)
+	if h := obj.getPrettyHostname(); h != "" {
+		propertyCheckOK, err := obj.updateHostnameProperty(hostnameObject, h, "PrettyHostname", "SetPrettyHostname", apply)
 		if err != nil {
 			return false, err
 		}
 		checkOK = checkOK && propertyCheckOK
 	}
-	if obj.StaticHostname != "" {
-		propertyCheckOK, err := obj.updateHostnameProperty(hostnameObject, obj.StaticHostname, "StaticHostname", "SetStaticHostname", apply)
+	if h := obj.getStaticHostname(); h != "" {
+		propertyCheckOK, err := obj.updateHostnameProperty(hostnameObject, h, "StaticHostname", "SetStaticHostname", apply)
 		if err != nil {
 			return false, err
 		}
 		checkOK = checkOK && propertyCheckOK
 	}
-	if obj.TransientHostname != "" {
-		propertyCheckOK, err := obj.updateHostnameProperty(hostnameObject, obj.TransientHostname, "Hostname", "SetHostname", apply)
+	if h := obj.getTransientHostname(); h != "" {
+		propertyCheckOK, err := obj.updateHostnameProperty(hostnameObject, h, "Hostname", "SetHostname", apply)
 		if err != nil {
 			return false, err
 		}
@@ -242,13 +295,13 @@ func (obj *HostnameRes) Cmp(r engine.Res) error {
 		return fmt.Errorf("not a %s", obj.Kind())
 	}
 
-	if obj.PrettyHostname != res.PrettyHostname {
+	if engineUtil.StrPtrCmp(obj.PrettyHostname, res.PrettyHostname) != nil {
 		return fmt.Errorf("the PrettyHostname differs")
 	}
-	if obj.StaticHostname != res.StaticHostname {
+	if engineUtil.StrPtrCmp(obj.StaticHostname, res.StaticHostname) != nil {
 		return fmt.Errorf("the StaticHostname differs")
 	}
-	if obj.TransientHostname != res.TransientHostname {
+	if engineUtil.StrPtrCmp(obj.TransientHostname, res.TransientHostname) != nil {
 		return fmt.Errorf("the TransientHostname differs")
 	}
 
@@ -271,9 +324,9 @@ func (obj *HostnameRes) UIDs() []engine.ResUID {
 	x := &HostnameUID{
 		BaseUID:           engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
 		name:              obj.Name(),
-		prettyHostname:    obj.PrettyHostname,
-		staticHostname:    obj.StaticHostname,
-		transientHostname: obj.TransientHostname,
+		prettyHostname:    obj.getPrettyHostname(),
+		staticHostname:    obj.getStaticHostname(),
+		transientHostname: obj.getTransientHostname(),
 	}
 	return []engine.ResUID{x}
 }

engine/resources/http.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify

engine/resources/http_flag.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -238,12 +238,8 @@ func (obj *HTTPFlagRes) Watch(ctx context.Context) error {
 
 // CheckApply never has anything to do for this resource, so it always succeeds.
 func (obj *HTTPFlagRes) CheckApply(ctx context.Context, apply bool) (bool, error) {
-	if obj.init.Debug {
-		obj.init.Logf("CheckApply")
-	}
-
 	if obj.init.Debug || true { // XXX: maybe we should always do this?
-		obj.init.Logf("CheckApply: value: %+v", obj.value)
+		obj.init.Logf("value: %+v", obj.value)
 	}
 
 	// TODO: can we send an empty (nil) value to show it has been removed?

engine/resources/http_proxy.go

@@ -1,5 +1,5 @@
 // Mgmt
-// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Copyright (C) James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
@@ -196,7 +196,8 @@ func (obj *HTTPProxyRes) serveHTTP(ctx context.Context, requestPath string) (han
 
 		// Tell the client right away, that we're working on things...
 		// TODO: Is this valuable to give us more time to block?
-		w.WriteHeader(http.StatusProcessing) // http 102, RFC 2518, 10.1
+		// NOTE: Using this header breaks wget2!
+		//w.WriteHeader(http.StatusProcessing) // http 102, RFC 2518, 10.1
 
 		response, err := client.Do(request) // (*Response, error)
 		if err != nil {