test: Don't fail on tag builds

This seems to be causing our failures with:

$ git fetch --unshallow
fatal: Couldn't find remote ref refs/heads/0.0.x

where x is some tag.

Hopefully this doesn't break the other use case we added this patch for!

.editorconfig

@@ -12,6 +12,9 @@ end_of_line = lf
 insert_final_newline = true
 trim_trailing_whitespace = true
 
+[*.sh]
+indent_style = tab
+
 [*.go]
 indent_style = tab
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,36 @@
+## Tips:
+
+* commit message titles must be in the form:
+```topic: Capitalized message with no trailing period```
+or:
+```topic, topic2: Capitalized message with no trailing period```
+
+* golang code must be formatted according to the standard, please run:
+```
+make gofmt		# formats the entire project correctly
+```
+or format a single golang file correctly:
+```
+gofmt -w yourcode.go
+```
+
+* please rebase your patch against current git master:
+```
+git checkout master
+git pull origin master
+git checkout your-feature
+git rebase master
+git push your-remote your-feature
+hub pull-request	# or submit with the github web ui
+```
+
+* after a patch review, please ping @purpleidea so we know to re-review:
+```
+# make changes based on reviews...
+git add -p		# add new changes
+git commit --amend	# combine with existing commit
+git push your-remote your-feature -f
+# now ping @purpleidea in the github PR since it doesn't notify us automatically
+```
+
+## Thanks for contributing to mgmt and welcome to the team!

.github/settings.yml

@@ -0,0 +1,96 @@
+# These settings are synced to GitHub by https://probot.github.io/apps/settings/
+
+repository:
+  # See https://developer.github.com/v3/repos/#edit for all available settings.
+
+  # The name of the repository. Changing this will rename the repository
+  name: mgmt
+
+  # A short description of the repository that will show up on GitHub
+  description: Next generation distributed, event-driven, parallel config management!
+
+  # A URL with more information about the repository
+  homepage: https://ttboj.wordpress.com/?s=mgmtconfig
+
+  # A comma-separated list of topics to set on the repository
+  topics: golang, go, configuration-management, config-management, devops, etcd, distributed-systems, graph-theory
+
+  # Either `true` to make the repository private, or `false` to make it public.
+  private: false
+
+  # Either `true` to enable issues for this repository, `false` to disable them.
+  has_issues: true
+
+  # Either `true` to enable projects for this repository, or `false` to disable them.
+  # If projects are disabled for the organization, passing `true` will cause an API error.
+  has_projects: false
+
+  # Either `true` to enable the wiki for this repository, `false` to disable it.
+  has_wiki: false
+
+  # Either `true` to enable downloads for this repository, `false` to disable them.
+  has_downloads: true
+
+  # Updates the default branch for this repository.
+  default_branch: master
+
+  # Either `true` to allow squash-merging pull requests, or `false` to prevent
+  # squash-merging.
+  allow_squash_merge: false
+
+  # Either `true` to allow merging pull requests with a merge commit, or `false`
+  # to prevent merging pull requests with merge commits.
+  allow_merge_commit: false
+
+  # Either `true` to allow rebase-merging pull requests, or `false` to prevent
+  # rebase-merging.
+  allow_rebase_merge: true
+
+# Labels: define labels for Issues and Pull Requests (in alphabetical order)
+labels:
+  - name: bug
+    color: fc2929
+  - name: confirmed
+    color: d93f0b
+  - name: design
+    color: 5319e7
+  - name: duplicate
+    color: cccccc
+  - name: enhancement
+    color: 84b6eb
+  - name: good first issue
+    color: 7057ff
+  - name: help wanted
+    color: 159818
+  - name: invalid
+    color: e6e6e6
+  - name: mgmtlove
+    color: e11d21
+  - name: question
+    color: cc317c
+  - name: wontfix
+    color: ffffff
+#  - name: first-timers-only
+#    # include the old name to rename an existing label
+#    oldname: Help Wanted
+
+# Collaborators: give specific users access to this repository.
+#collaborators:
+#  - username: purpleidea
+#    # Note: Only valid on organization-owned repositories.
+#    # The permission to grant the collaborator. Can be one of:
+#    # * `pull` - can pull, but not push to or administer this repository.
+#    # * `push` - can pull and push, but not administer this repository.
+#    # * `admin` - can pull, push and administer this repository.
+#    permission: push
+
+#  - username: hubot
+#    permission: pull
+
+# NOTE: The APIs needed for teams are not supported yet by GitHub Apps
+# https://developer.github.com/v3/apps/available-endpoints/
+#teams:
+#  - name: core
+#    permission: admin
+#  - name: docs
+#    permission: push

.gitignore

@@ -2,9 +2,11 @@
 .omv/
 .ssh/
 .vagrant/
+.envrc
 old/
 tmp/
 *_stringer.go
+bindata/*.go
 mgmt
 mgmt.static
 mgmt.iml

.gitmodules

@@ -16,3 +16,9 @@
 [submodule "vendor/honnef.co/go/augeas"]
 	path = vendor/honnef.co/go/augeas
 	url = https://github.com/dominikh/go-augeas/
+[submodule "vendor/github.com/grpc-ecosystem/go-grpc-prometheus"]
+	path = vendor/github.com/grpc-ecosystem/go-grpc-prometheus
+	url = https://github.com/grpc-ecosystem/go-grpc-prometheus
+[submodule "vendor/github.com/ugorji/go"]
+	path = vendor/github.com/ugorji/go
+	url = https://github.com/ugorji/go

.travis.yml

@@ -1,14 +1,14 @@
 language: go
 go:
-  - 1.6.x
-  - 1.7.x
   - 1.8.x
+  - 1.9.x
   - tip
 go_import_path: github.com/purpleidea/mgmt
 sudo: true
 dist: trusty
 before_install:
   - sudo apt update
+  - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
   - git fetch --unshallow
 install: 'make deps'
 script: 'make test'
@@ -16,7 +16,7 @@ matrix:
   fast_finish: true
   allow_failures:
     - go: tip
-    - go: 1.8.x
+    - go: 1.9.x
 notifications:
   irc:
     channels:

COPYING

@@ -1,5 +1,5 @@
-                    GNU AFFERO GENERAL PUBLIC LICENSE
+                    GNU GENERAL PUBLIC LICENSE
-                       Version 3, 19 November 2007
+                       Version 3, 29 June 2007
 
  Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
@@ -7,15 +7,17 @@
 
                             Preamble
 
-  The GNU Affero General Public License is a free, copyleft license for
+  The GNU General Public License is a free, copyleft license for
-software and other kinds of works, specifically designed to ensure
+software and other kinds of works.
-cooperation with the community in the case of network server software.
 
   The licenses for most software and other practical works are designed
 to take away your freedom to share and change the works.  By contrast,
-our General Public Licenses are intended to guarantee your freedom to
+the GNU General Public License is intended to guarantee your freedom to
 share and change all versions of a program--to make sure it remains free
-software for all its users.
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
 
   When we speak of free software, we are referring to freedom, not
 price.  Our General Public Licenses are designed to make sure that you
@@ -24,34 +26,44 @@ them if you wish), that you receive source code or can get it if you
 want it, that you can change the software or use pieces of it in new
 free programs, and that you know you can do these things.
 
-  Developers that use our General Public Licenses protect your rights
+  To protect your rights, we need to prevent others from denying you
-with two steps: (1) assert copyright on the software, and (2) offer
+these rights or asking you to surrender the rights.  Therefore, you have
-you this License which gives you legal permission to copy, distribute
+certain responsibilities if you distribute copies of the software, or if
-and/or modify the software.
+you modify it: responsibilities to respect the freedom of others.
 
-  A secondary benefit of defending all users' freedom is that
+  For example, if you distribute copies of such a program, whether
-improvements made in alternate versions of the program, if they
+gratis or for a fee, you must pass on to the recipients the same
-receive widespread use, become available for other developers to
+freedoms that you received.  You must make sure that they, too, receive
-incorporate.  Many developers of free software are heartened and
+or can get the source code.  And you must show them these terms so they
-encouraged by the resulting cooperation.  However, in the case of
+know their rights.
-software used on network servers, this result may fail to come about.
-The GNU General Public License permits making a modified version and
-letting the public access it on a server without ever releasing its
-source code to the public.
 
-  The GNU Affero General Public License is designed specifically to
+  Developers that use the GNU GPL protect your rights with two steps:
-ensure that, in such cases, the modified source code becomes available
+(1) assert copyright on the software, and (2) offer you this License
-to the community.  It requires the operator of a network server to
+giving you legal permission to copy, distribute and/or modify it.
-provide the source code of the modified version running there to the
-users of that server.  Therefore, public use of a modified version, on
-a publicly accessible server, gives the public access to the source
-code of the modified version.
 
-  An older license, called the Affero General Public License and
+  For the developers' and authors' protection, the GPL clearly explains
-published by Affero, was designed to accomplish similar goals.  This is
+that there is no warranty for this free software.  For both users' and
-a different license, not a version of the Affero GPL, but Affero has
+authors' sake, the GPL requires that modified versions be marked as
-released a new version of the Affero GPL which permits relicensing under
+changed, so that their problems will not be attributed erroneously to
-this license.
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
 
   The precise terms and conditions for copying, distribution and
 modification follow.
@@ -60,7 +72,7 @@ modification follow.
 
   0. Definitions.
 
-  "This License" refers to version 3 of the GNU Affero General Public License.
+  "This License" refers to version 3 of the GNU General Public License.
 
   "Copyright" also means copyright-like laws that apply to other kinds of
 works, such as semiconductor masks.
@@ -537,45 +549,35 @@ to collect a royalty for further conveying from those to whom you convey
 the Program, the only way you could satisfy both those terms and this
 License would be to refrain entirely from conveying the Program.
 
-  13. Remote Network Interaction; Use with the GNU General Public License.
+  13. Use with the GNU Affero General Public License.
-
-  Notwithstanding any other provision of this License, if you modify the
-Program, your modified version must prominently offer all users
-interacting with it remotely through a computer network (if your version
-supports such interaction) an opportunity to receive the Corresponding
-Source of your version by providing access to the Corresponding Source
-from a network server at no charge, through some standard or customary
-means of facilitating copying of software.  This Corresponding Source
-shall include the Corresponding Source for any work covered by version 3
-of the GNU General Public License that is incorporated pursuant to the
-following paragraph.
 
   Notwithstanding any other provision of this License, you have
 permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
+under version 3 of the GNU Affero General Public License into a single
 combined work, and to convey the resulting work.  The terms of this
 License will continue to apply to the part which is the covered work,
-but the work with which it is combined will remain governed by version
+but the special requirements of the GNU Affero General Public License,
-3 of the GNU General Public License.
+section 13, concerning interaction through a network will apply to the
+combination as such.
 
   14. Revised Versions of this License.
 
   The Free Software Foundation may publish revised and/or new versions of
-the GNU Affero General Public License from time to time.  Such new versions
+the GNU General Public License from time to time.  Such new versions will
-will be similar in spirit to the present version, but may differ in detail to
+be similar in spirit to the present version, but may differ in detail to
 address new problems or concerns.
 
   Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU Affero General
+Program specifies that a certain numbered version of the GNU General
 Public License "or any later version" applies to it, you have the
 option of following the terms and conditions either of that numbered
 version or of any later version published by the Free Software
 Foundation.  If the Program does not specify a version number of the
-GNU Affero General Public License, you may choose any version ever published
+GNU General Public License, you may choose any version ever published
 by the Free Software Foundation.
 
   If the Program specifies that a proxy can decide which future
-versions of the GNU Affero General Public License can be used, that proxy's
+versions of the GNU General Public License can be used, that proxy's
 public statement of acceptance of a version permanently authorizes you
 to choose that version for the Program.
 
@@ -633,29 +635,40 @@ the "copyright" line and a pointer to where the full notice is found.
     Copyright (C) <year>  <name of author>
 
     This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
+    it under the terms of the GNU General Public License as published by
     the Free Software Foundation, either version 3 of the License, or
     (at your option) any later version.
 
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
+    GNU General Public License for more details.
 
-    You should have received a copy of the GNU Affero General Public License
+    You should have received a copy of the GNU General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
-  If your software can interact with users remotely through a computer
+  If the program does terminal interaction, make it output a short
-network, you should also make sure that it provides a way for users to
+notice like this when it starts in an interactive mode:
-get its source.  For example, if your program is a web application, its
+
-interface could display a "Source" link that leads users to an archive
+    <program>  Copyright (C) <year>  <name of author>
-of the code.  There are many ways you could offer source, and different
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-solutions will be better for different programs; see section 13 for the
+    This is free software, and you are welcome to redistribute it
-specific requirements.
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
 
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
+For more information on this, and how to apply and follow the GNU GPL, see
 <http://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.

COPYRIGHT

@@ -1,16 +1,16 @@
 Mgmt
-Copyright (C) 2013-2017+ James Shubin and the project contributors
+Copyright (C) 2013-2018+ James Shubin and the project contributors
 Written by James Shubin <james@shubin.ca> and the project contributors
 
 This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
+it under the terms of the GNU General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.
 
 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
+GNU General Public License for more details.
 
-You should have received a copy of the GNU Affero General Public License
+You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.

Makefile

@@ -1,28 +1,28 @@
 # Mgmt
-# Copyright (C) 2013-2017+ James Shubin and the project contributors
+# Copyright (C) 2013-2018+ James Shubin and the project contributors
 # Written by James Shubin <james@shubin.ca> and the project contributors
 #
 # This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
+# it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
+# GNU General Public License for more details.
 #
-# You should have received a copy of the GNU Affero General Public License
+# You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 SHELL = /usr/bin/env bash
-.PHONY: all art cleanart version program path deps run race generate build clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms copr
+.PHONY: all art cleanart version program path deps run race bindata generate build clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms copr
-.SILENT: clean
+.SILENT: clean bindata
 
+GO_FILES := $(shell find . -name '*.go')
 SVERSION := $(or $(SVERSION),$(shell git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --tags --dirty --always))
 VERSION := $(or $(VERSION),$(shell git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --tags --abbrev=0))
 PROGRAM := $(shell echo $(notdir $(CURDIR)) | cut -f1 -d"-")
-OLDGOLANG := $(shell go version | grep -E 'go1.3|go1.4')
 ifeq ($(VERSION),$(SVERSION))
 	RELEASE = 1
 else
@@ -101,40 +101,36 @@ run:
 race:
 	find . -maxdepth 1 -type f -name '*.go' -not -name '*_test.go' | xargs go run -race -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)"
 
+# generate go files from non-go source
+bindata:
+	$(MAKE) --quiet -C bindata
+
 generate:
 	go generate
 
-build: $(PROGRAM)
+build: bindata $(PROGRAM)
 
-$(PROGRAM): main.go
+$(PROGRAM): $(GO_FILES)
 	@echo "Building: $(PROGRAM), version: $(SVERSION)..."
-ifneq ($(OLDGOLANG),)
-	@# avoid equals sign in old golang versions eg in: -X foo=bar
-	time go build -ldflags "-X main.program $(PROGRAM) -X main.version $(SVERSION)" -o $(PROGRAM) $(BUILD_FLAGS);
-else
 	time go build -i -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)" -o $(PROGRAM) $(BUILD_FLAGS);
-endif
 
-$(PROGRAM).static: main.go
+$(PROGRAM).static: $(GO_FILES)
 	@echo "Building: $(PROGRAM).static, version: $(SVERSION)..."
 	go generate
-ifneq ($(OLDGOLANG),)
+	go build -a -installsuffix cgo -tags netgo -ldflags '-extldflags "-static" -X main.program=$(PROGRAM) -X main.version=$(SVERSION) -s -w' -o $(PROGRAM).static $(BUILD_FLAGS);
-	@# avoid equals sign in old golang versions eg in: -X foo=bar
-	go build -a -installsuffix cgo -tags netgo -ldflags '-extldflags "-static" -X main.program $(PROGRAM) -X main.version $(SVERSION)' -o $(PROGRAM).static $(BUILD_FLAGS);
-else
-	go build -a -installsuffix cgo -tags netgo -ldflags '-extldflags "-static" -X main.program=$(PROGRAM) -X main.version=$(SVERSION)' -o $(PROGRAM).static $(BUILD_FLAGS);
-endif
 
 clean:
 	[ ! -e $(PROGRAM) ] || rm $(PROGRAM)
 	rm -f *_stringer.go	# generated by `go generate`
 	rm -f *_mock.go		# generated by `go generate`
 
-test:
+test: bindata
 	./test.sh
 
 gofmt:
-	find . -maxdepth 3 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -exec gofmt -w {} \;
+	# TODO: remove gofmt once goimports has a -s option
+	find . -maxdepth 3 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -exec gofmt -s -w {} \;
+	find . -maxdepth 3 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -exec goimports -w {} \;
 
 yamlfmt:
 	find . -maxdepth 3 -type f -name '*.yaml' -not -path './old/*' -not -path './tmp/*' -not -path './omv.yaml' -exec ruby -e "require 'yaml'; x=YAML.load_file('{}').to_yaml.each_line.map(&:rstrip).join(10.chr)+10.chr; File.open('{}', 'w').write x" \;

README.md

@@ -22,6 +22,7 @@ Mgmt is a fairly new project.
 We're working towards being minimally useful for production environments.
 We aren't feature complete for what we'd consider a 1.x release yet.
 With your help you'll be able to influence our design and get us there sooner!
+Interested developers should read the [quick start guide](docs/quick-start-guide.md).
 
 ## Documentation:
 Please read, enjoy and help improve our documentation!
@@ -30,6 +31,7 @@ Please read, enjoy and help improve our documentation!
 |---|---|
 | [general documentation](docs/documentation.md) | for everyone |
 | [quick start guide](docs/quick-start-guide.md) | for mgmt developers |
+| [frequently asked questions](docs/faq.md) | for everyone |
 | [resource guide](docs/resource-guide.md) | for mgmt developers |
 | [godoc API reference](https://godoc.org/github.com/purpleidea/mgmt) | for mgmt developers |
 | [prometheus guide](docs/prometheus.md) | for everyone |
@@ -40,9 +42,9 @@ Please ask in the [community](#community)!
 If you have a well phrased question that might benefit others, consider asking it by sending a patch to the documentation [FAQ](https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md#usage-and-frequently-asked-questions) section. I'll merge your question, and a patch with the answer!
 
 ## Roadmap:
+Feel free to grab one of the straightforward [#mgmtlove](https://github.com/purpleidea/mgmt/labels/mgmtlove) issues if you're a first time contributor to the project or if you're unsure about what to hack on!
 Please see: [TODO.md](TODO.md) for a list of upcoming work and TODO items.
 Please get involved by working on one of these items or by suggesting something else!
-Feel free to grab one of the straightforward [#mgmtlove](https://github.com/purpleidea/mgmt/labels/mgmtlove) issues if you're a first time contributor to the project or if you're unsure about what to hack on!
 
 ## Bugs:
 Please set the `DEBUG` constant in [main.go](https://github.com/purpleidea/mgmt/blob/master/main.go) to `true`, and post the logs when you report the [issue](https://github.com/purpleidea/mgmt/issues).
@@ -53,33 +55,7 @@ Feel free to read my article on [debugging golang programs](https://ttboj.wordpr
 We'd love to have your patches! Please send them by email, or as a pull request.
 
 ## On the web:
-| Author | Format | Subject |
+[Read what people are saying and publishing about mgmt!](docs/on-the-web.md)
-|---|---|---|
-| James Shubin | blog | [Next generation configuration mgmt](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/) |
-| James Shubin | video | [Introductory recording from DevConf.cz 2016](https://www.youtube.com/watch?v=GVhpPF0j-iE&html5=1) |
-| James Shubin | video | [Introductory recording from CfgMgmtCamp.eu 2016](https://www.youtube.com/watch?v=fNeooSiIRnA&html5=1) |
-| Julian Dunn | video | [On mgmt at CfgMgmtCamp.eu 2016](https://www.youtube.com/watch?v=kfF9IATUask&t=1949&html5=1) |
-| Walter Heck | slides | [On mgmt at CfgMgmtCamp.eu 2016](http://www.slideshare.net/olindata/configuration-management-time-for-a-4th-generation/3) |
-| Marco Marongiu | blog | [On mgmt](http://syslog.me/2016/02/15/leap-or-die/) |
-| Felix Frank | blog | [From Catalog To Mgmt (on puppet to mgmt "transpiling")](https://ffrank.github.io/features/2016/02/18/from-catalog-to-mgmt/) |
-| James Shubin | blog | [Automatic edges in mgmt (...and the pkg resource)](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/) |
-| James Shubin | blog | [Automatic grouping in mgmt](https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/) |
-| John Arundel | tweet | [“Puppet’s days are numbered.”](https://twitter.com/bitfield/status/732157519142002688) |
-| Felix Frank | blog | [Puppet, Meet Mgmt (on puppet to mgmt internals)](https://ffrank.github.io/features/2016/06/12/puppet,-meet-mgmt/) |
-| Felix Frank | blog | [Puppet Powered Mgmt (puppet to mgmt tl;dr)](https://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/) |
-| James Shubin | blog | [Automatic clustering in mgmt](https://ttboj.wordpress.com/2016/06/20/automatic-clustering-in-mgmt/) |
-| James Shubin | video | [Recording from CoreOSFest 2016](https://www.youtube.com/watch?v=KVmDCUA42wc&html5=1) |
-| James Shubin | video | [Recording from DebConf16](http://meetings-archive.debian.net/pub/debian-meetings/2016/debconf16/Next_Generation_Config_Mgmt.webm) ([Slides](https://annex.debconf.org//debconf-share/debconf16/slides/15-next-generation-config-mgmt.pdf)) |
-| Felix Frank | blog | [Edging It All In (puppet and mgmt edges)](https://ffrank.github.io/features/2016/07/12/edging-it-all-in/) |
-| Felix Frank | blog | [Translating All The Things (puppet to mgmt translation warnings)](https://ffrank.github.io/features/2016/08/19/translating-all-the-things/) |
-| James Shubin | video | [Recording from systemd.conf 2016](https://www.youtube.com/watch?v=jB992Zb3nH0&html5=1) |
-| James Shubin | blog | [Remote execution in mgmt](https://ttboj.wordpress.com/2016/10/07/remote-execution-in-mgmt/) |
-| James Shubin | video | [Recording from High Load Strategy 2016](https://vimeo.com/191493409) |
-| James Shubin | video | [Recording from NLUUG 2016](https://www.youtube.com/watch?v=MmpwOQAb_SE&html5=1) |
-| James Shubin | blog | [Send/Recv in mgmt](https://ttboj.wordpress.com/2016/12/07/sendrecv-in-mgmt/) |
-| James Shubin | blog | [Metaparameters in mgmt](https://ttboj.wordpress.com/2017/03/01/metaparameters-in-mgmt/) |
-| James Shubin | video | [Recording from Incontro DevOps 2017](https://vimeo.com/212241877) |
-| Yves Brissaud | blog | [mgmt aux HumanTalks Grenoble (french)](http://log.winsos.net/2017/04/12/mgmt-aux-human-talks-grenoble.html) |
 
 ##
 

Vagrantfile

@@ -6,13 +6,16 @@ Vagrant.configure(2) do |config|
 	config.vm.synced_folder ".", "/vagrant", disabled: true
 
 	config.vm.define "mgmt-dev" do |instance|
-		instance.vm.box = "fedora/24-cloud-base"
+		instance.vm.box = "fedora/26-cloud-base"
 	end
 
 	config.vm.provider "virtualbox" do |v|
 		v.memory = 1536
 		v.cpus = 2
 	end
+	config.vm.provider "libvirt" do |v|
+		v.memory = 2048
+	end
 
 	config.vm.provision "file", source: "vagrant/motd", destination: ".motd"
 	config.vm.provision "shell", inline: "cp ~vagrant/.motd /etc/motd"

bindata/Makefile

@@ -0,0 +1,33 @@
+# Mgmt
+# Copyright (C) 2013-2018+ James Shubin and the project contributors
+# Written by James Shubin <james@shubin.ca> and the project contributors
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# The bindata target generates go files from any source defined below. To use
+# the files, import the "bindata" package and use:
+# `bytes, err := bindata.Asset("FILEPATH")`
+# where FILEPATH is the path of the original input file relative to `bindata/`.
+
+.PHONY: build
+default: build
+
+build: bindata.go
+
+# add more input files as dependencies at the end here...
+bindata.go: ../COPYING
+	# go-bindata --pkg bindata -o {OUTPUT} {INPUT}
+	go-bindata --pkg bindata -o ./$@ $^
+	# gofmt the output file
+	gofmt -s -w $@

converger/converger.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 // Package converger is a facility for reporting the converged state.
@@ -85,7 +85,7 @@ type cuid struct {
 }
 
 // NewConverger builds a new converger struct.
-func NewConverger(timeout int, stateFn func(bool) error) *converger {
+func NewConverger(timeout int, stateFn func(bool) error) Converger {
 	return &converger{
 		timeout: timeout,
 		stateFn: stateFn,

doc.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 // Package main provides the main entrypoint for using the `mgmt` software.

docker/Dockerfile

@@ -1,10 +1,10 @@
-FROM golang:1.6.2
+FROM golang:1.8
 
 MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
 
 # Set the reset cache variable
 # Read more here: http://czerasz.com/2014/11/13/docker-tip-and-tricks/#use-refreshedat-variable-for-better-cache-control
-ENV REFRESHED_AT 2016-05-10
+ENV REFRESHED_AT 2017-11-16
 
 # Update the package list to be able to use required packages
 RUN apt-get update

docker/Dockerfile.development

@@ -1,10 +1,10 @@
-FROM golang:1.6.2
+FROM golang:1.8
 
 MAINTAINER Michał Czeraszkiewicz <contact@czerasz.com>
 
 # Set the reset cache variable
 # Read more here: http://czerasz.com/2014/11/13/docker-tip-and-tricks/#use-refreshedat-variable-for-better-cache-control
-ENV REFRESHED_AT 2016-05-14
+ENV REFRESHED_AT 2017-11-16
 
 RUN apt-get update
 
@@ -27,5 +27,8 @@ WORKDIR /home/$USER_NAME/mgmt
 # Install dependencies
 RUN make deps
 
+# Chown $GOPATH
+RUN chown -R ${USER_ID}:${GROUP_ID} /go
+
 # Change user
 USER ${USER_NAME}

docs/conf.py

@@ -51,7 +51,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'mgmt'
-copyright = u'2013-2017+ James Shubin and the project contributors'
+copyright = u'2013-2018+ James Shubin and the project contributors'
 author = u'James Shubin'
 
 # The version info for the project you're documenting, acts as replacement for

docs/documentation.md

@@ -1,9 +1,4 @@
-# mgmt
+# General documentation
-
-Available from:
-[https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/)
-
-This documentation is available in: [Markdown](https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md) or [PDF](https://pdfdoc-purpleidea.rhcloud.com/pdf/https://github.com/purpleidea/mgmt/blob/master/docs/documentation.md) format.
 
 ## Overview
 
@@ -26,16 +21,12 @@ For more information, you may like to read some blog posts from the author:
 * [Send/Recv in mgmt](https://ttboj.wordpress.com/2016/12/07/sendrecv-in-mgmt/)
 * [Metaparameters in mgmt](https://ttboj.wordpress.com/2017/03/01/metaparameters-in-mgmt/)
 
-There is also an [introductory video](http://meetings-archive.debian.net/pub/debian-meetings/2016/debconf16/Next_Generation_Config_Mgmt.webm) available.
+There is also an [introductory video](https://www.youtube.com/watch?v=LkEtBVLfygE&html5=1) available.
-Older videos and other material [is available](https://github.com/purpleidea/mgmt/#on-the-web).
+Older videos and other material [is available](on-the-web.md).
 
 ## Setup
 
-During this prototype phase, the tool can be run out of the source directory.
+You'll probably want to read the [quick start guide](quick-start-guide.md) to get going.
-You'll probably want to use ```./run.sh run --yaml examples/graph1.yaml``` to
-get started. Beware that this _can_ cause data loss. Understand what you're
-doing first, or perform these actions in a virtual environment such as the one
-provided by [Oh-My-Vagrant](https://github.com/purpleidea/oh-my-vagrant).
 
 ## Features
 
@@ -162,255 +153,6 @@ For more details and caveats see [Puppet.md](Puppet.md).
 An introductory post on the Puppet support is on
 [Felix's blog](http://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/).
 
-## Resources
-
-This section lists all the built-in resources and their properties. The
-resource primitives in `mgmt` are typically more powerful than resources in
-other configuration management systems because they can be event based which
-lets them respond in real-time to converge to the desired state. This property
-allows you to build more complex resources that you probably hadn't considered
-in the past.
-
-In addition to the resource specific properties, there are resource properties
-(otherwise known as parameters) which can apply to every resource. These are
-called [meta parameters](#meta-parameters) and are listed separately. Certain
-meta parameters aren't very useful when combined with certain resources, but
-in general, it should be fairly obvious, such as when combining the `noop` meta
-parameter with the [Noop](#Noop) resource.
-
-* [Augeas](#Augeas): Manipulate files using augeas.
-* [Exec](#Exec): Execute shell commands on the system.
-* [File](#File): Manage files and directories.
-* [Hostname](#Hostname): Manages the hostname on the system.
-* [KV](#KV): Set a key value pair in our shared world database.
-* [Msg](#Msg): Send log messages.
-* [Noop](#Noop): A simple resource that does nothing.
-* [Nspawn](#Nspawn): Manage systemd-machined nspawn containers.
-* [Password](#Password): Create random password strings.
-* [Pkg](#Pkg):  Manage system packages with PackageKit.
-* [Svc](#Svc): Manage system systemd services.
-* [Timer](#Timer): Manage system systemd services.
-* [Virt](#Virt): Manage virtual machines with libvirt.
-
-
-### Augeas
-
-The augeas resource uses [augeas](http://augeas.net/) commands to manipulate
-files.
-
-### Exec
-
-The exec resource can execute commands on your system.
-
-### File
-
-The file resource manages files and directories. In `mgmt`, directories are
-identified by a trailing slash in their path name. File have no such slash.
-
-It has the following properties:
-
-- `path`: file path (directories have a trailing slash here)
-- `content`: raw file content
-- `state`: either `exists` (the default value) or `absent`
-- `mode`: octal unix file permissions
-- `owner`: username or uid for the file owner
-- `group`: group name or gid for the file group
-
-#### Path
-
-The path property specifies the file or directory that we are managing.
-
-#### Content
-
-The content property is a string that specifies the desired file contents.
-
-#### Source
-
-The source property points to a source file or directory path that we wish to
-copy over and use as the desired contents for our resource.
-
-#### State
-
-The state property describes the action we'd like to apply for the resource. The
-possible values are: `exists` and `absent`.
-
-#### Recurse
-
-The recurse property limits whether file resource operations should recurse into
-and monitor directory contents with a depth greater than one.
-
-#### Force
-
-The force property is required if we want the file resource to be able to change
-a file into a directory or vice-versa. If such a change is needed, but the force
-property is not set to `true`, then this file resource will error.
-
-### Hostname
-
-The hostname resource manages static, transient/dynamic and pretty hostnames
-on the system and watches them for changes.
-
-#### static_hostname
-The static hostname is the one configured in /etc/hostname or a similar
-file.
-It is chosen by the local user. It is not always in sync with the current
-host name as returned by the gethostname() system call.
-
-#### transient_hostname
-The transient / dynamic hostname is the one configured via the kernel's
-sethostbyname().
-It can be different from the static hostname in case DHCP or mDNS have been
-configured to change the name based on network information.
-
-#### pretty_hostname
-The pretty hostname is a free-form UTF8 host name for presentation to the user.
-
-#### hostname
-Hostname is the fallback value for all 3 fields above, if only `hostname` is
-specified, it will set all 3 fields to this value.
-
-### KV
-
-The KV resource sets a key and value pair in the global world database. This is
-quite useful for setting a flag after a number of resources have run. It will
-ignore database updates to the value that are greater in compare order than the
-requested key if the `SkipLessThan` parameter is set to true. If we receive a
-refresh, then the stored value will be reset to the requested value even if the
-stored value is greater.
-
-#### Key
-The string key used to store the key.
-
-#### Value
-The string value to set. This can also be set via Send/Recv.
-
-#### SkipLessThan
-If this parameter is set to `true`, then it will ignore updating the value as
-long as the database versions are greater than the requested value. The compare
-operation used is based on the `SkipCmpStyle` parameter.
-
-#### SkipCmpStyle
-By default this converts the string values to integers and compares them as you
-would expect.
-
-### Msg
-
-The msg resource sends messages to the main log, or an external service such
-as systemd's journal.
-
-### Noop
-
-The noop resource does absolutely nothing. It does have some utility in testing
-`mgmt` and also as a placeholder in the resource graph.
-
-### Nspawn
-
-The nspawn resource is used to manage systemd-machined style containers.
-
-### Password
-
-The password resource can generate a random string to be used as a password. It
-will re-generate the password if it receives a refresh notification.
-
-### Pkg
-
-The pkg resource is used to manage system packages. This resource works on many
-different distributions because it uses the underlying packagekit facility which
-supports different backends for different environments. This ensures that we
-have great Debian (deb/dpkg) and Fedora (rpm/dnf) support simultaneously.
-
-### Svc
-
-The service resource is still very WIP. Please help us my improving it!
-
-### Timer
-
-This resource needs better documentation. Please help us my improving it!
-
-### Virt
-
-The virt resource can manage virtual machines via libvirt.
-
-## Usage and frequently asked questions
-(Send your questions as a patch to this FAQ! I'll review it, merge it, and
-respond by commit with the answer.)
-
-### Why did you start this project?
-
-I wanted a next generation config management solution that didn't have all of
-the design flaws or limitations that the current generation of tools do, and no
-tool existed!
-
-### Why did you use etcd? What about consul?
-
-Etcd and consul are both written in golang, which made them the top two
-contenders for my prototype. Ultimately a choice had to be made, and etcd was
-chosen, but it was also somewhat arbitrary. If there is available interest,
-good reasoning, *and* patches, then we would consider either switching or
-supporting both, but this is not a high priority at this time.
-
-### Can I use an existing etcd cluster instead of the automatic embedded servers?
-
-Yes, it's possible to use an existing etcd cluster instead of the automatic,
-elastic embedded etcd servers. To do so, simply point to the cluster with the
-`--seeds` variable, the same way you would if you were seeding a new member to
-an existing mgmt cluster.
-
-The downside to this approach is that you won't benefit from the automatic
-elastic nature of the embedded etcd servers, and that you're responsible if you
-accidentally break your etcd cluster, or if you use an unsupported version.
-
-### What does the error message about an inconsistent dataDir mean?
-
-If you get an error message similar to:
-
-```
-Etcd: Connect: CtxError...
-Etcd: CtxError: Reason: CtxDelayErr(5s): No endpoints available yet!
-Etcd: Connect: Endpoints: []
-Etcd: The dataDir (/var/lib/mgmt/etcd) might be inconsistent or corrupt.
-```
-
-This happens when there are a series of fatal connect errors in a row. This can
-happen when you start `mgmt` using a dataDir that doesn't correspond to the
-current cluster view. As a result, the embedded etcd server never finishes
-starting up, and as a result, a default endpoint never gets added. The solution
-is to either reconcile the mistake, and if there is no important data saved, you
-can remove the etcd dataDir. This is typically `/var/lib/mgmt/etcd/member/`.
-
-### Why do resources have both a `Compare` method and an `IFF` (on the UID) method?
-
-The `Compare()` methods are for determining if two resources are effectively the
-same, which is used to make graph change delta's efficient. This is when we want
-to change from the current running graph to a new graph, but preserve the common
-vertices. Since we want to make this process efficient, we only update the parts
-that are different, and leave everything else alone. This `Compare()` method can
-tell us if two resources are the same.
-
-The `IFF()` method is part of the whole UID system, which is for discerning if a
-resource meets the requirements another expects for an automatic edge. This is
-because the automatic edge system assumes a unified UID pattern to test for
-equality. In the future it might be helpful or sane to merge the two similar
-comparison functions although for now they are separate because they are
-actually answer different questions.
-
-### Did you know that there is a band named `MGMT`?
-
-I didn't realize this when naming the project, and it is accidental. After much
-anguishing, I chose the name because it was short and I thought it was
-appropriately descriptive. If you need a less ambiguous search term or phrase,
-you can try using `mgmtconfig` or `mgmt config`.
-
-### You didn't answer my question, or I have a question!
-
-It's best to ask on [IRC](https://webchat.freenode.net/?channels=#mgmtconfig)
-to see if someone can help you. Once we get a big enough community going, we'll
-add a mailing list. If you don't get any response from the above, you can
-contact me through my [technical blog](https://ttboj.wordpress.com/contact/)
-and I'll do my best to help. If you have a good question, please add it as a
-patch to this documentation. I'll merge your question, and add a patch with the
-answer!
-
 ## Reference
 Please note that there are a number of undocumented options. For more
 information on these options, please view the source at:
@@ -635,7 +377,7 @@ To report any bugs, please file a ticket at: [https://github.com/purpleidea/mgmt
 
 ## Authors
 
-Copyright (C) 2013-2017+ James Shubin and the project contributors
+Copyright (C) 2013-2018+ James Shubin and the project contributors
 
 Please see the
 [AUTHORS](https://github.com/purpleidea/mgmt/tree/master/AUTHORS) file

docs/faq.md

@@ -0,0 +1,87 @@
+# Frequently asked questions
+
+(Send your questions as a patch to this FAQ! I'll review it, merge it, and
+respond by commit with the answer.)
+
+### Why did you start this project?
+
+I wanted a next generation config management solution that didn't have all of
+the design flaws or limitations that the current generation of tools do, and no
+tool existed!
+
+### Why did you use etcd? What about consul?
+
+Etcd and consul are both written in golang, which made them the top two
+contenders for my prototype. Ultimately a choice had to be made, and etcd was
+chosen, but it was also somewhat arbitrary. If there is available interest,
+good reasoning, *and* patches, then we would consider either switching or
+supporting both, but this is not a high priority at this time.
+
+### Can I use an existing etcd cluster instead of the automatic embedded servers?
+
+Yes, it's possible to use an existing etcd cluster instead of the automatic,
+elastic embedded etcd servers. To do so, simply point to the cluster with the
+`--seeds` variable, the same way you would if you were seeding a new member to
+an existing mgmt cluster.
+
+The downside to this approach is that you won't benefit from the automatic
+elastic nature of the embedded etcd servers, and that you're responsible if you
+accidentally break your etcd cluster, or if you use an unsupported version.
+
+### What does the error message about an inconsistent dataDir mean?
+
+If you get an error message similar to:
+
+```
+Etcd: Connect: CtxError...
+Etcd: CtxError: Reason: CtxDelayErr(5s): No endpoints available yet!
+Etcd: Connect: Endpoints: []
+Etcd: The dataDir (/var/lib/mgmt/etcd) might be inconsistent or corrupt.
+```
+
+This happens when there are a series of fatal connect errors in a row. This can
+happen when you start `mgmt` using a dataDir that doesn't correspond to the
+current cluster view. As a result, the embedded etcd server never finishes
+starting up, and as a result, a default endpoint never gets added. The solution
+is to either reconcile the mistake, and if there is no important data saved, you
+can remove the etcd dataDir. This is typically `/var/lib/mgmt/etcd/member/`.
+
+### Why do resources have both a `Compare` method and an `IFF` (on the UID) method?
+
+The `Compare()` methods are for determining if two resources are effectively the
+same, which is used to make graph change delta's efficient. This is when we want
+to change from the current running graph to a new graph, but preserve the common
+vertices. Since we want to make this process efficient, we only update the parts
+that are different, and leave everything else alone. This `Compare()` method can
+tell us if two resources are the same.
+
+The `IFF()` method is part of the whole UID system, which is for discerning if a
+resource meets the requirements another expects for an automatic edge. This is
+because the automatic edge system assumes a unified UID pattern to test for
+equality. In the future it might be helpful or sane to merge the two similar
+comparison functions although for now they are separate because they are
+actually answer different questions.
+
+### Does this support Windows? OSX? GNU Hurd?
+
+Mgmt probably works best on Linux, because that's what most developers use for
+serious automation workloads. Support for non-Linux operating systems isn't a
+high priority of mine, but we're happy to accept patches for missing features
+or resources that you think would make sense on your favourite platform.
+
+### Did you know that there is a band named `MGMT`?
+
+I didn't realize this when naming the project, and it is accidental. After much
+anguishing, I chose the name because it was short and I thought it was
+appropriately descriptive. If you need a less ambiguous search term or phrase,
+you can try using `mgmtconfig` or `mgmt config`.
+
+### You didn't answer my question, or I have a question!
+
+It's best to ask on [IRC](https://webchat.freenode.net/?channels=#mgmtconfig)
+to see if someone can help you. Once we get a big enough community going, we'll
+add a mailing list. If you don't get any response from the above, you can
+contact me through my [technical blog](https://ttboj.wordpress.com/contact/)
+and I'll do my best to help. If you have a good question, please add it as a
+patch to this documentation. I'll merge your question, and add a patch with the
+answer!

docs/on-the-web.md

@@ -0,0 +1,36 @@
+# On the web
+
+Here is a list of places mgmt has appeared on the web. Feel free to send a patch
+if we missed something that you think is relevant!
+
+## Links
+| Author | Format | Subject |
+|---|---|---|
+| James Shubin | blog | [Next generation configuration mgmt](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/) |
+| James Shubin | video | [Introductory recording from DevConf.cz 2016](https://www.youtube.com/watch?v=GVhpPF0j-iE&html5=1) |
+| James Shubin | video | [Introductory recording from CfgMgmtCamp.eu 2016](https://www.youtube.com/watch?v=fNeooSiIRnA&html5=1) |
+| Julian Dunn | video | [On mgmt at CfgMgmtCamp.eu 2016](https://www.youtube.com/watch?v=kfF9IATUask&t=1949&html5=1) |
+| Walter Heck | slides | [On mgmt at CfgMgmtCamp.eu 2016](http://www.slideshare.net/olindata/configuration-management-time-for-a-4th-generation/3) |
+| Marco Marongiu | blog | [On mgmt](http://syslog.me/2016/02/15/leap-or-die/) |
+| Felix Frank | blog | [From Catalog To Mgmt (on puppet to mgmt "transpiling")](https://ffrank.github.io/features/2016/02/18/from-catalog-to-mgmt/) |
+| James Shubin | blog | [Automatic edges in mgmt (...and the pkg resource)](https://ttboj.wordpress.com/2016/03/14/automatic-edges-in-mgmt/) |
+| James Shubin | blog | [Automatic grouping in mgmt](https://ttboj.wordpress.com/2016/03/30/automatic-grouping-in-mgmt/) |
+| John Arundel | tweet | [“Puppet’s days are numbered.”](https://twitter.com/bitfield/status/732157519142002688) |
+| Felix Frank | blog | [Puppet, Meet Mgmt (on puppet to mgmt internals)](https://ffrank.github.io/features/2016/06/12/puppet,-meet-mgmt/) |
+| Felix Frank | blog | [Puppet Powered Mgmt (puppet to mgmt tl;dr)](https://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/) |
+| James Shubin | blog | [Automatic clustering in mgmt](https://ttboj.wordpress.com/2016/06/20/automatic-clustering-in-mgmt/) |
+| James Shubin | video | [Recording from CoreOSFest 2016](https://www.youtube.com/watch?v=KVmDCUA42wc&html5=1) |
+| James Shubin | video | [Recording from DebConf16](http://meetings-archive.debian.net/pub/debian-meetings/2016/debconf16/Next_Generation_Config_Mgmt.webm) ([Slides](https://annex.debconf.org//debconf-share/debconf16/slides/15-next-generation-config-mgmt.pdf)) |
+| Felix Frank | blog | [Edging It All In (puppet and mgmt edges)](https://ffrank.github.io/features/2016/07/12/edging-it-all-in/) |
+| Felix Frank | blog | [Translating All The Things (puppet to mgmt translation warnings)](https://ffrank.github.io/features/2016/08/19/translating-all-the-things/) |
+| James Shubin | video | [Recording from systemd.conf 2016](https://www.youtube.com/watch?v=jB992Zb3nH0&html5=1) |
+| James Shubin | blog | [Remote execution in mgmt](https://ttboj.wordpress.com/2016/10/07/remote-execution-in-mgmt/) |
+| James Shubin | video | [Recording from High Load Strategy 2016](https://vimeo.com/191493409) |
+| James Shubin | video | [Recording from NLUUG 2016](https://www.youtube.com/watch?v=MmpwOQAb_SE&html5=1) |
+| James Shubin | blog | [Send/Recv in mgmt](https://ttboj.wordpress.com/2016/12/07/sendrecv-in-mgmt/) |
+| Julien Pivotto | blog | [Augeas resource for mgmt](https://roidelapluie.be/blog/2017/02/14/mgmt-augeas/) |
+| James Shubin | blog | [Metaparameters in mgmt](https://ttboj.wordpress.com/2017/03/01/metaparameters-in-mgmt/) |
+| James Shubin | video | [Recording from Incontro DevOps 2017](https://vimeo.com/212241877) |
+| Yves Brissaud | blog | [mgmt aux HumanTalks Grenoble (french)](http://log.winsos.net/2017/04/12/mgmt-aux-human-talks-grenoble.html) |
+| James Shubin | video | [Recording from OSDC Berlin 2017](https://www.youtube.com/watch?v=LkEtBVLfygE&html5=1) |
+| Jonathan Gold | blog | [AWS:EC2 in mgmt](http://jonathangold.ca/awsec2-in-mgmt/) |

docs/prometheus.md

@@ -30,7 +30,7 @@ Here is a list of the metrics we provide:
 - `mgmt_resources_total`: The number of resources that mgmt is managing
 - `mgmt_checkapply_total`: The number of CheckApply's that mgmt has run
 - `mgmt_failures_total`: The number of resources that have failed
-- `mgmt_failures_current`: The number of resources that have failed
+- `mgmt_failures`: The number of resources that have failed
 - `mgmt_graph_start_time_seconds`: Start time of the current graph since unix epoch in seconds
 
 For each metric, you will get some extra labels:
@@ -63,4 +63,4 @@ We do not have grafana dashboards yet. Patches welcome!
 
 [pgc]: https://github.com/prometheus/client_golang/blob/master/prometheus/go_collector.go
 [etcdm]: https://coreos.com/etcd/docs/latest/metrics.html
-[pd]: https://github.com/prometheus/prometheus/wiki/Default-port-allocation
+[pd]: https://github.com/prometheus/prometheus/wiki/Default-port-allocations

docs/quick-start-guide.md

@@ -2,65 +2,22 @@
 
 ## Introduction
 This guide is intended for developers. Once `mgmt` is minimally viable, we'll
-publish a quick start guide for users too. In the meantime, please contribute!
+publish a quick start guide for users too. If you're brand new to `mgmt`, it's
-If you're brand new to `mgmt`, it's probably a good idea to start by reading the
+probably a good idea to start by reading the
 [introductory article](https://ttboj.wordpress.com/2016/01/18/next-generation-configuration-mgmt/)
-or to watch an [introductory video](https://github.com/purpleidea/mgmt/#on-the-web).
+or to watch an [introductory video](https://www.youtube.com/watch?v=LkEtBVLfygE&html5=1).
 Once you're familiar with the general idea, please start hacking...
 
-## Vagrant
-If you would like to avoid doing the following steps manually, we have prepared
-a [Vagrant](https://www.vagrantup.com/) environment for your convenience. From
-the project directory, run a `vagrant up`, and then a `vagrant status`. From
-there, you can `vagrant ssh` into the `mgmt` machine. The MOTD will explain the
-rest.
-
-## Dependencies
-Software projects have a few different kinds of dependencies. There are _build_
-dependencies, _runtime_ dependencies, and additionally, a few extra dependencies
-required for running the _test_ suite.
-
-### Build
-* `golang` 1.6 or higher (required, available in most distros)
-* golang libraries (required, available with `go get ./...`) a partial list includes:
-```
-github.com/coreos/etcd/client
-gopkg.in/yaml.v2
-gopkg.in/fsnotify.v1
-github.com/urfave/cli
-github.com/coreos/go-systemd/dbus
-github.com/coreos/go-systemd/util
-github.com/libvirt/libvirt-go
-```
-* `stringer` (optional), available as a package on some platforms, otherwise via `go get`
-```
-golang.org/x/tools/cmd/stringer
-```
-* `pandoc` (optional), for building a pdf of the documentation
-
-### Runtime
-A relatively modern GNU/Linux system should be able to run `mgmt` without any
-problems. Since `mgmt` runs as a single statically compiled binary, all of the
-library dependencies are included. It is expected, that certain advanced
-resources require host specific facilities to work. These requirements are
-listed below:
-
-| Resource | Dependency        | Version |
-|----------|-------------------|---------|
-| file     | inotify           | ?       |
-| hostname | systemd-hostnamed | ?       |
-| nspawn   | systemd-nspawn    | ?       |
-| pkg      | packagekitd       | ?       |
-| svc      | systemd           | ?       |
-| virt     | libvirtd          | ?       |
-
-For building a visual representation of the graph, `graphviz` is required.
-
-### Testing
-* golint `github.com/golang/lint/golint`
-
 ## Quick start
-* Make sure you have golang version 1.6 or greater installed.
+
+### Installing golang
+* You need golang version 1.8 or greater installed.
+** To install on rpm style systems: `sudo dnf install golang`
+** To install on apt style systems: `sudo apt install golang`
+* You can run `go version` to check the golang version.
+* If your distro is tool old, you may need to [download](https://golang.org/dl/) a newer golang version.
+
+### Setting up golang
 * If you do not have a GOPATH yet, create one and export it:
 ```
 mkdir $HOME/gopath
@@ -68,7 +25,9 @@ export GOPATH=$HOME/gopath
 ```
 * You might also want to add the GOPATH to your `~/.bashrc` or `~/.profile`.
 * For more information you can read the [GOPATH documentation](https://golang.org/cmd/go/#hdr-GOPATH_environment_variable).
-* Next download the mgmt code base, and switch to that directory:
+
+### Getting the mgmt code and dependencies
+* Download the `mgmt` code into the GOPATH, and switch to that directory:
 ```
 mkdir -p $GOPATH/src/github.com/purpleidea/
 cd $GOPATH/src/github.com/purpleidea/
@@ -77,15 +36,64 @@ cd $GOPATH/src/github.com/purpleidea/mgmt
 ```
 * Run `make deps` to install system and golang dependencies. Take a look at `misc/make-deps.sh` for details.
 * Run `make build` to get a freshly built `mgmt` binary.
+
+### Running mgmt
 * Run `time ./mgmt run --yaml examples/graph0.yaml --converged-timeout=5 --tmp-prefix` to try out a very simple example!
 * To run continuously in the default mode of operation, omit the `--converged-timeout` option.
-* Have fun hacking on our future technology!
+* Look in that example file that you ran to see if you can figure out what it did!
+* The yaml frontend is provided as a developer tool to test the engine until the language is ready.
+* Have fun hacking on our future technology and get involved to shape the project!
 
 ## Examples
-Please look in the [examples/](../examples/) folder for some examples!
+Please look in the [examples/](../examples/) folder for some more examples!
 
-## Installation
+## Vagrant
+If you would like to avoid doing the above steps manually, we have prepared a
+[Vagrant](https://www.vagrantup.com/) environment for your convenience. From the
+project directory, run a `vagrant up`, and then a `vagrant status`. From there,
+you can `vagrant ssh` into the `mgmt` machine. The MOTD will explain the rest.
+
+## Information about dependencies
+Software projects have a few different kinds of dependencies. There are _build_
+dependencies, _runtime_ dependencies, and additionally, a few extra dependencies
+required for running the _test_ suite.
+
+### Build
+* `golang` 1.8 or higher (required, available in some distros and distributed
+  as a binary officially by [golang.org](https://golang.org/dl/))
+
+### Runtime
+A relatively modern GNU/Linux system should be able to run `mgmt` without any
+problems. Since `mgmt` runs as a single statically compiled binary, all of the
+library dependencies are included. It is expected, that certain advanced
+resources require host specific facilities to work. These requirements are
+listed below:
+
+| Resource | Dependency        | Version                     | Check version with                                        |
+|----------|-------------------|-----------------------------|-----------------------------------------------------------|
+| augeas   | augeas-devel      | `augeas 1.6` or greater     | `dnf info augeas-devel` or `apt-cache show libaugeas-dev` |
+| file     | inotify           | `Linux 2.6.27` or greater   | `uname -a`                                                |
+| hostname | systemd-hostnamed | `systemd 25` or greater     | `systemctl --version`                                     |
+| nspawn   | systemd-nspawn    | `systemd ???` or greater    | `systemctl --version`                                     |
+| pkg      | packagekitd       | `packagekit 1.x` or greater | `pkcon --version`                                         |
+| svc      | systemd           | `systemd ???` or greater    | `systemctl --version`                                     |
+| virt     | libvirt-devel     | `libvirt 1.2.0` or greater  | `dnf info libvirt-devel` or `apt-cache show libvirt-dev`  |
+| virt     | libvirtd          | `libvirt 1.2.0` or greater  | `libvirtd --version`                                      |
+
+For building a visual representation of the graph, `graphviz` is required.
+
+To build `mgmt` without augeas support please run:
+`GOTAGS='noaugeas' make build`
+
+To build `mgmt` without libvirt support please run:
+`GOTAGS='novirt' make build`
+
+To build `mgmt` without augeas or libvirt support please run:
+`GOTAGS='noaugeas novirt' make build`
+
+## Binary Package Installation
 Installation of `mgmt` from distribution packages currently needs improvement.
+They are not always up-to-date with git master and as such are not recommended.
 At the moment we have:
 * [COPR](https://copr.fedoraproject.org/coprs/purpleidea/mgmt/)
 * [Arch](https://aur.archlinux.org/packages/mgmt/)

docs/resource-guide.md

@@ -73,14 +73,13 @@ Init() error
 ```
 
 This is called to initialize the resource. If something goes wrong, it should
-return an error. It should set the resource `kind`, do any resource specific
+return an error. It should do any resource specific work, and finish by calling
-work, and finish by calling the `Init` method of the base resource.
+the `Init` method of the base resource.
 
 #### Example
 ```golang
 // Init initializes the Foo resource.
 func (obj *FooRes) Init() error {
-	obj.BaseRes.Kind = "foo" // must lower case resource kind
 	// run the resource specific initialization, and error if anything fails
 	if some_error {
 		return err // something went wrong!
@@ -399,9 +398,9 @@ UnmarshalYAML(unmarshal func(interface{}) error) error // optional
 ```
 
 This is optional, but recommended for any resource that will have a YAML
-accessible struct, and an entry in the `GraphConfig` struct. It is not required
+accessible struct. It is not required because to do so would mean that
-because to do so would mean that third-party or custom resources (such as those
+third-party or custom resources (such as those someone writes to use with
-someone writes to use with `libmgmt`) would have to implement this needlessly.
+`libmgmt`) would have to implement this needlessly.
 
 The signature intentionally matches what is required to satisfy the `go-yaml`
 [Unmarshaler](https://godoc.org/gopkg.in/yaml.v2#Unmarshaler) interface.
@@ -453,35 +452,15 @@ type FooRes struct {
 }
 ```
 
-### YAML
+### Resource registration
-In addition to labelling your resource struct with YAML fields, you must also
+All resources must be registered with the engine so that they can be found. This
-add an entry to the internal `GraphConfig` struct. It is a fairly straight
+also ensures they can be encoded and decoded. Make sure to include the following
-forward one line patch.
+code snippet for this to work.
 
 ```golang
-type GraphConfig struct {
-// [snip...]
-	Resources struct {
-		Noop []*resources.NoopRes `yaml:"noop"`
-		File []*resources.FileRes `yaml:"file"`
-		// [snip...]
-		Foo []*resources.FooRes `yaml:"foo"` // tada :)
-	}
-}
-```
-
-It's also recommended that you add the [UnmarshalYAML](#unmarshalyaml) method to
-your resources so that unspecified values are given sane defaults.
-
-### Gob registration
-All resources must be registered with the `golang` _gob_ module so that they can
-be encoded and decoded. Make sure to include the following code snippet for this
-to work.
-
-```golang
-import "encoding/gob"
 func init() { // special golang method that runs once
-	gob.Register(&FooRes{}) // substitude your resource here
+	// set your resource kind and struct here (the kind must be lower case)
+	RegisterResource("foo", func() Res { return &FooRes{} })
 }
 ```
 

docs/resources.md

@@ -0,0 +1,169 @@
+# Resources
+
+Here we list all the built-in resources and their properties. The resource
+primitives in `mgmt` are typically more powerful than resources in other
+configuration management systems because they can be event based which lets them
+respond in real-time to converge to the desired state. This property allows you
+to build more complex resources that you probably hadn't considered in the past.
+
+In addition to the resource specific properties, there are resource properties
+(otherwise known as parameters) which can apply to every resource. These are
+called [meta parameters](documentation.md#meta-parameters) and are listed
+separately. Certain meta parameters aren't very useful when combined with
+certain resources, but in general, it should be fairly obvious, such as when
+combining the `noop` meta parameter with the [Noop](#Noop) resource.
+
+You might want to look at the [generated documentation](https://godoc.org/github.com/purpleidea/mgmt/resources)
+for more up-to-date information about these resources.
+
+* [Augeas](#Augeas): Manipulate files using augeas.
+* [Exec](#Exec): Execute shell commands on the system.
+* [File](#File): Manage files and directories.
+* [Hostname](#Hostname): Manages the hostname on the system.
+* [KV](#KV): Set a key value pair in our shared world database.
+* [Msg](#Msg): Send log messages.
+* [Noop](#Noop): A simple resource that does nothing.
+* [Nspawn](#Nspawn): Manage systemd-machined nspawn containers.
+* [Password](#Password): Create random password strings.
+* [Pkg](#Pkg):  Manage system packages with PackageKit.
+* [Svc](#Svc): Manage system systemd services.
+* [Timer](#Timer): Manage system systemd services.
+* [Virt](#Virt): Manage virtual machines with libvirt.
+
+## Augeas
+
+The augeas resource uses [augeas](http://augeas.net/) commands to manipulate
+files.
+
+## Exec
+
+The exec resource can execute commands on your system.
+
+## File
+
+The file resource manages files and directories. In `mgmt`, directories are
+identified by a trailing slash in their path name. File have no such slash.
+
+It has the following properties:
+
+- `path`: file path (directories have a trailing slash here)
+- `content`: raw file content
+- `state`: either `exists` (the default value) or `absent`
+- `mode`: octal unix file permissions
+- `owner`: username or uid for the file owner
+- `group`: group name or gid for the file group
+
+### Path
+
+The path property specifies the file or directory that we are managing.
+
+### Content
+
+The content property is a string that specifies the desired file contents.
+
+### Source
+
+The source property points to a source file or directory path that we wish to
+copy over and use as the desired contents for our resource.
+
+### State
+
+The state property describes the action we'd like to apply for the resource. The
+possible values are: `exists` and `absent`.
+
+### Recurse
+
+The recurse property limits whether file resource operations should recurse into
+and monitor directory contents with a depth greater than one.
+
+### Force
+
+The force property is required if we want the file resource to be able to change
+a file into a directory or vice-versa. If such a change is needed, but the force
+property is not set to `true`, then this file resource will error.
+
+## Hostname
+
+The hostname resource manages static, transient/dynamic and pretty hostnames
+on the system and watches them for changes.
+
+### static_hostname
+The static hostname is the one configured in /etc/hostname or a similar
+file.
+It is chosen by the local user. It is not always in sync with the current
+host name as returned by the gethostname() system call.
+
+### transient_hostname
+The transient / dynamic hostname is the one configured via the kernel's
+sethostbyname().
+It can be different from the static hostname in case DHCP or mDNS have been
+configured to change the name based on network information.
+
+### pretty_hostname
+The pretty hostname is a free-form UTF8 host name for presentation to the user.
+
+### hostname
+Hostname is the fallback value for all 3 fields above, if only `hostname` is
+specified, it will set all 3 fields to this value.
+
+## KV
+
+The KV resource sets a key and value pair in the global world database. This is
+quite useful for setting a flag after a number of resources have run. It will
+ignore database updates to the value that are greater in compare order than the
+requested key if the `SkipLessThan` parameter is set to true. If we receive a
+refresh, then the stored value will be reset to the requested value even if the
+stored value is greater.
+
+### Key
+The string key used to store the key.
+
+### Value
+The string value to set. This can also be set via Send/Recv.
+
+### SkipLessThan
+If this parameter is set to `true`, then it will ignore updating the value as
+long as the database versions are greater than the requested value. The compare
+operation used is based on the `SkipCmpStyle` parameter.
+
+### SkipCmpStyle
+By default this converts the string values to integers and compares them as you
+would expect.
+
+## Msg
+
+The msg resource sends messages to the main log, or an external service such
+as systemd's journal.
+
+## Noop
+
+The noop resource does absolutely nothing. It does have some utility in testing
+`mgmt` and also as a placeholder in the resource graph.
+
+## Nspawn
+
+The nspawn resource is used to manage systemd-machined style containers.
+
+## Password
+
+The password resource can generate a random string to be used as a password. It
+will re-generate the password if it receives a refresh notification.
+
+## Pkg
+
+The pkg resource is used to manage system packages. This resource works on many
+different distributions because it uses the underlying packagekit facility which
+supports different backends for different environments. This ensures that we
+have great Debian (deb/dpkg) and Fedora (rpm/dnf) support simultaneously.
+
+## Svc
+
+The service resource is still very WIP. Please help us my improving it!
+
+## Timer
+
+This resource needs better documentation. Please help us my improving it!
+
+## Virt
+
+The virt resource can manage virtual machines via libvirt.

etcd/etcd.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 // TODO: Add TTL's (eg: volunteering)
@@ -181,6 +181,8 @@ type EmbdEtcd struct { // EMBeddeD etcd
 	endpoints           etcdtypes.URLsMap // map of servers a client could connect to
 	clientURLs          etcdtypes.URLs    // locations to listen for clients if i am a server
 	serverURLs          etcdtypes.URLs    // locations to listen for servers if i am a server (peer)
+	advertiseClientURLs etcdtypes.URLs    // client urls to advertise
+	advertiseServerURLs etcdtypes.URLs    // server urls to advertise
 	noServer            bool              // disable all server peering if true
 
 	// local tracked state
@@ -208,7 +210,7 @@ type EmbdEtcd struct { // EMBeddeD etcd
 }
 
 // NewEmbdEtcd creates the top level embedded etcd struct client and server obj.
-func NewEmbdEtcd(hostname string, seeds, clientURLs, serverURLs etcdtypes.URLs, noServer bool, idealClusterSize uint16, flags Flags, prefix string, converger converger.Converger) *EmbdEtcd {
+func NewEmbdEtcd(hostname string, seeds, clientURLs, serverURLs, advertiseClientURLs, advertiseServerURLs etcdtypes.URLs, noServer bool, idealClusterSize uint16, flags Flags, prefix string, converger converger.Converger) *EmbdEtcd {
 	endpoints := make(etcdtypes.URLsMap)
 	if hostname == seedSentinel { // safety
 		return nil
@@ -233,6 +235,8 @@ func NewEmbdEtcd(hostname string, seeds, clientURLs, serverURLs etcdtypes.URLs,
 		endpoints:           endpoints,
 		clientURLs:          clientURLs,
 		serverURLs:          serverURLs,
+		advertiseClientURLs: advertiseClientURLs,
+		advertiseServerURLs: advertiseServerURLs,
 		noServer:            noServer,
 
 		idealClusterSize: idealClusterSize,
@@ -396,9 +400,13 @@ func (obj *EmbdEtcd) Startup() error {
 	// if i am alone and will have to be a server...
 	if !obj.noServer && bootstrapping {
 		log.Printf("Etcd: Bootstrapping...")
+		surls := obj.serverURLs
+		if len(obj.advertiseServerURLs) > 0 {
+			surls = obj.advertiseServerURLs
+		}
 		// give an initial value to the obj.nominate map we keep in sync
 		// this emulates Nominate(obj, obj.hostname, obj.serverURLs)
-		obj.nominated[obj.hostname] = obj.serverURLs // initial value
+		obj.nominated[obj.hostname] = surls // initial value
 		// NOTE: when we are stuck waiting for the server to start up,
 		// it is probably happening on this call right here...
 		obj.nominateCallback(nil) // kick this off once
@@ -407,8 +415,12 @@ func (obj *EmbdEtcd) Startup() error {
 	// self volunteer
 	if !obj.noServer && len(obj.serverURLs) > 0 {
 		// we run this in a go routine because it blocks waiting for server
+		surls := obj.serverURLs
+		if len(obj.advertiseServerURLs) > 0 {
+			surls = obj.advertiseServerURLs
+		}
 		log.Printf("Etcd: Startup: Volunteering...")
-		go Volunteer(obj, obj.serverURLs)
+		go Volunteer(obj, surls)
 	}
 
 	if bootstrapping {
@@ -529,8 +541,9 @@ func (obj *EmbdEtcd) CtxError(ctx context.Context, err error) (context.Context,
 	if obj.ctxErr != nil { // stop on permanent error
 		return ctx, obj.ctxErr
 	}
-	const ctxErr = "ctxErr"
+	type ctxKey string // use a non-basic type as ctx key (str can conflict)
-	const ctxIter = "ctxIter"
+	const ctxErr ctxKey = "ctxErr"
+	const ctxIter ctxKey = "ctxIter"
 	expBackoff := func(tmin, texp, iter, tmax int) time.Duration {
 		// https://en.wikipedia.org/wiki/Exponential_backoff
 		// tmin <= texp^iter - 1 <= tmax // TODO: check my math
@@ -1435,14 +1448,21 @@ func (obj *EmbdEtcd) nominateCallback(re *RE) error {
 			// client connects to one of the obj.endpoints servers...
 			log.Printf("Etcd: Addresses are: %s", addresses)
 
+			surls := obj.serverURLs
+			if len(obj.advertiseServerURLs) > 0 {
+				surls = obj.advertiseServerURLs
+			}
 			// XXX: just put this wherever for now so we don't block
 			// nominate self so "member" list is correct for peers to see
-			Nominate(obj, obj.hostname, obj.serverURLs)
+			Nominate(obj, obj.hostname, surls)
 			// XXX: if this fails, where will we retry this part ?
 		}
 
 		// advertise client urls
 		if curls := obj.clientURLs; len(curls) > 0 {
+			if len(obj.advertiseClientURLs) > 0 {
+				curls = obj.advertiseClientURLs
+			}
 			// XXX: don't advertise local addresses! 127.0.0.1:2381 doesn't really help remote hosts
 			// XXX: but sometimes this is what we want... hmmm how do we decide? filter on callback?
 			AdvertiseEndpoints(obj, curls)
@@ -1646,14 +1666,23 @@ func (obj *EmbdEtcd) StartServer(newCluster bool, peerURLsMap etcdtypes.URLsMap)
 		initialPeerURLsMap[memberName] = peerURLs
 	}
 
+	aCUrls := obj.clientURLs
+	if len(obj.advertiseClientURLs) > 0 {
+		aCUrls = obj.advertiseClientURLs
+	}
+	aPUrls := peerURLs
+	if len(obj.advertiseServerURLs) > 0 {
+		aPUrls = obj.advertiseServerURLs
+	}
+
 	// embed etcd
 	cfg := embed.NewConfig()
 	cfg.Name = memberName // hostname
 	cfg.Dir = obj.dataDir
-	cfg.ACUrls = obj.clientURLs
-	cfg.APUrls = peerURLs
 	cfg.LCUrls = obj.clientURLs
 	cfg.LPUrls = peerURLs
+	cfg.ACUrls = aCUrls
+	cfg.APUrls = aPUrls
 	cfg.StrictReconfigCheck = false // XXX: workaround https://github.com/coreos/etcd/issues/6305
 
 	cfg.InitialCluster = initialPeerURLsMap.String() // including myself!

etcd/methods.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package etcd

etcd/resources.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package etcd
@@ -171,7 +171,6 @@ func GetResources(obj *EmbdEtcd, hostnameFilter, kindFilter []string) ([]resourc
 		}
 
 		if obj, err := resources.B64ToRes(val); err == nil {
-			obj.SetKind(kind) // cheap init
 			log.Printf("Etcd: Get: (Hostname, Kind, Name): (%s, %s, %s)", hostname, kind, name)
 			resourceList = append(resourceList, obj)
 		} else {

etcd/str.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package etcd

etcd/strmap.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package etcd

etcd/world.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package etcd

event/event.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 // Package event provides some primitives that are used for message passing.

examples/autoedges1.yaml

@@ -13,7 +13,5 @@ resources:
     meta:
       autoedge: true
     path: "/tmp/foo/"
-    content: |
-      i am f2
     state: exists
 edges: []

examples/autoedges4.yaml

@@ -0,0 +1,19 @@
+---
+graph: mygraph
+resources:
+  user:
+  - name: edgeuser
+    state: absent
+    gid: 10000
+  - name: edgeuser2
+    state: exists
+    group: edgegroup
+    groups: [edgegroup2, edgegroup3]
+  group:
+  - name: edgegroup
+    state: exists
+    gid: 10000
+  - name: edgegroup2
+    state: exists
+  - name: edgegroup3
+    state: exists

examples/autoedges5.yaml

@@ -0,0 +1,21 @@
+---
+graph: mygraph
+resources:
+  pkg:
+  - name: httpd
+    meta:
+      autoedge: true
+      noop: true
+    state: installed
+  exec:
+  - name: pkg10
+    cmd: /usr/bin/apachectl status
+    shell: ''
+    timeout: 0
+    watchcmd: ''
+    watchshell: ''
+    ifcmd: ''
+    ifshell: ''
+    pollint: 0
+    state: present
+edges: []

examples/aws_ec2_1.yaml

@@ -0,0 +1,10 @@
+---
+graph: mygraph
+resources:
+  aws:ec2:
+  - name: ec2example
+    region: ca-central-1
+    type: t2.micro
+    imageid: ami-5ac17f3e
+    state: running
+edges: []

examples/graph0.hcl

@@ -0,0 +1,14 @@
+resource "file" "file1" {
+  path = "/tmp/mgmt-hello-world"
+  content = "hello, world"
+  state = "exists"
+  depends_on = ["noop.noop1", "exec.sleep"]
+}
+
+resource "noop" "noop1" {
+  test = "nil"
+}
+
+resource "exec" "sleep" {
+ cmd = "sleep 10s"
+}

examples/graph1.hcl

@@ -0,0 +1,4 @@
+resource "exec" "exec1" {
+  cmd = "cat /tmp/mgmt-hello-world"
+  state =  "present"
+}

examples/group1.yaml

@@ -0,0 +1,8 @@
+---
+graph: mygraph
+resources:
+  group:
+  - name: testgroup
+    state: exists
+    gid: 10000
+edges: []

examples/hil.hcl

@@ -0,0 +1,9 @@
+resource "file" "file1" {
+  path = "/tmp/mgmt-hello-world"
+  content = "${exec.sleep.Output}"
+  state = "exists"
+}
+
+resource "exec" "sleep" {
+ cmd = "echo hello"
+}

examples/lib/exec-send-recv.go

@@ -61,12 +61,12 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 		return nil, err
 	}
 
-	// FIXME: these are being specified temporarily until it's the default!
 	metaparams := resources.DefaultMetaParams
 
 	exec1 := &resources.ExecRes{
 		BaseRes: resources.BaseRes{
 			Name:       "exec1",
+			Kind:       "exec",
 			MetaParams: metaparams,
 		},
 		Cmd:   "echo hello world && echo goodbye world 1>&2", // to stdout && stderr
@@ -77,6 +77,7 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 	output := &resources.FileRes{
 		BaseRes: resources.BaseRes{
 			Name:       "output",
+			Kind:       "file",
 			MetaParams: metaparams,
 			// send->recv!
 			Recv: map[string]*resources.Send{
@@ -92,6 +93,7 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 	stdout := &resources.FileRes{
 		BaseRes: resources.BaseRes{
 			Name:       "stdout",
+			Kind:       "file",
 			MetaParams: metaparams,
 			// send->recv!
 			Recv: map[string]*resources.Send{
@@ -107,6 +109,7 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 	stderr := &resources.FileRes{
 		BaseRes: resources.BaseRes{
 			Name:       "stderr",
+			Kind:       "file",
 			MetaParams: metaparams,
 			// send->recv!
 			Recv: map[string]*resources.Send{
@@ -229,10 +232,7 @@ func Run() error {
 		}
 	}()
 
-	if err := obj.Run(); err != nil {
+	return obj.Run()
-		return err
-	}
-	return nil
 }
 
 func main() {

examples/lib/libmgmt-subgraph0.go

@@ -63,6 +63,7 @@ func (obj *MyGAPI) subGraph() (*pgraph.Graph, error) {
 	f1 := &resources.FileRes{
 		BaseRes: resources.BaseRes{
 			Name:       "file1",
+			Kind:       "file",
 			MetaParams: metaparams,
 		},
 		Path: "/tmp/mgmt/sub1",
@@ -74,6 +75,7 @@ func (obj *MyGAPI) subGraph() (*pgraph.Graph, error) {
 	n1 := &resources.NoopRes{
 		BaseRes: resources.BaseRes{
 			Name:       "noop1",
+			Kind:       "noop",
 			MetaParams: metaparams,
 		},
 	}
@@ -93,7 +95,6 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 		return nil, err
 	}
 
-	// FIXME: these are being specified temporarily until it's the default!
 	metaparams := resources.DefaultMetaParams
 
 	content := "I created a subgraph!\n"
@@ -238,10 +239,7 @@ func Run() error {
 		}
 	}()
 
-	if err := obj.Run(); err != nil {
+	return obj.Run()
-		return err
-	}
-	return nil
 }
 
 func main() {

examples/lib/libmgmt-subgraph1.go

@@ -61,13 +61,13 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 		return nil, err
 	}
 
-	// FIXME: these are being specified temporarily until it's the default!
 	metaparams := resources.DefaultMetaParams
 
 	content := "I created a subgraph!\n"
 	f0 := &resources.FileRes{
 		BaseRes: resources.BaseRes{
 			Name:       "README",
+			Kind:       "file",
 			MetaParams: metaparams,
 		},
 		Path:    "/tmp/mgmt/README",
@@ -86,6 +86,7 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 	f1 := &resources.FileRes{
 		BaseRes: resources.BaseRes{
 			Name:       "file1",
+			Kind:       "file",
 			MetaParams: metaparams,
 		},
 		Path: "/tmp/mgmt/sub1",
@@ -97,6 +98,7 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 	n1 := &resources.NoopRes{
 		BaseRes: resources.BaseRes{
 			Name:       "noop1",
+			Kind:       "noop",
 			MetaParams: metaparams,
 		},
 	}
@@ -110,6 +112,7 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 	subGraphRes0 := &resources.GraphRes{ // TODO: should we name this SubGraphRes ?
 		BaseRes: resources.BaseRes{
 			Name:       "subgraph1",
+			Kind:       "graph",
 			MetaParams: metaparams,
 		},
 		Graph: subGraph,
@@ -226,10 +229,7 @@ func Run() error {
 		}
 	}()
 
-	if err := obj.Run(); err != nil {
+	return obj.Run()
-		return err
-	}
-	return nil
 }
 
 func main() {

examples/lib/libmgmt1.go

@@ -57,13 +57,13 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 		return nil, fmt.Errorf("libmgmt: MyGAPI is not initialized")
 	}
 
-	n1 := &resources.NoopRes{
+	n1, err := resources.NewNamedResource("noop", "noop1")
-		BaseRes: resources.BaseRes{
+	if err != nil {
-			Name:       "noop1",
+		return nil, err
-			MetaParams: resources.DefaultMetaParams,
-		},
 	}
 
+	// NOTE: This is considered the legacy method to build graphs. Avoid
+	// importing the legacy `yamlgraph` lib if possible for custom graphs.
 	// we can still build a graph via the yaml method
 	gc := &yamlgraph.GraphConfig{
 		Graph: obj.Name,
@@ -72,7 +72,7 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 			Exec:  []*resources.ExecRes{},
 			File:  []*resources.FileRes{},
 			Msg:   []*resources.MsgRes{},
-			Noop:  []*resources.NoopRes{n1},
+			Noop:  []*resources.NoopRes{n1.(*resources.NoopRes)},
 			Pkg:   []*resources.PkgRes{},
 			Svc:   []*resources.SvcRes{},
 			Timer: []*resources.TimerRes{},
@@ -190,10 +190,7 @@ func Run() error {
 		}
 	}()
 
-	if err := obj.Run(); err != nil {
+	return obj.Run()
-		return err
-	}
-	return nil
 }
 
 func main() {

examples/lib/libmgmt2.go

@@ -65,11 +65,9 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 	}
 	var vertex pgraph.Vertex
 	for i := uint(0); i < obj.Count; i++ {
-		n := &resources.NoopRes{
+		n, err := resources.NewNamedResource("noop", fmt.Sprintf("noop%d", i))
-			BaseRes: resources.BaseRes{
+		if err != nil {
-				Name:       fmt.Sprintf("noop%d", i),
+			return nil, err
-				MetaParams: resources.DefaultMetaParams,
-			},
 		}
 		g.AddVertex(n)
 		if i > 0 {
@@ -186,10 +184,7 @@ func Run(count uint) error {
 		}
 	}()
 
-	if err := obj.Run(); err != nil {
+	return obj.Run()
-		return err
-	}
-	return nil
 }
 
 func main() {

examples/lib/libmgmt3.go

@@ -61,13 +61,13 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 		return nil, err
 	}
 
-	// FIXME: these are being specified temporarily until it's the default!
 	metaparams := resources.DefaultMetaParams
 
 	content := "Delete me to trigger a notification!\n"
 	f0 := &resources.FileRes{
 		BaseRes: resources.BaseRes{
 			Name:       "README",
+			Kind:       "file",
 			MetaParams: metaparams,
 		},
 		Path:    "/tmp/mgmt/README",
@@ -80,6 +80,7 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 	p1 := &resources.PasswordRes{
 		BaseRes: resources.BaseRes{
 			Name:       "password1",
+			Kind:       "password",
 			MetaParams: metaparams,
 		},
 		Length: 8,    // generated string will have this many characters
@@ -90,6 +91,7 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 	f1 := &resources.FileRes{
 		BaseRes: resources.BaseRes{
 			Name:       "file1",
+			Kind:       "file",
 			MetaParams: metaparams,
 			// send->recv!
 			Recv: map[string]*resources.Send{
@@ -106,6 +108,7 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
 	n1 := &resources.NoopRes{
 		BaseRes: resources.BaseRes{
 			Name:       "noop1",
+			Kind:       "noop",
 			MetaParams: metaparams,
 		},
 	}
@@ -231,10 +234,7 @@ func Run() error {
 		}
 	}()
 
-	if err := obj.Run(); err != nil {
+	return obj.Run()
-		return err
-	}
-	return nil
 }
 
 func main() {

examples/longpoll/redirect-client.go

@@ -0,0 +1,52 @@
+// This is an example longpoll client. The connection to the corresponding
+// server initiates a request on a "Watch". It then waits until a redirect is
+// received from the server which indicates that the watch is ready. To signal
+// than an event on this watch has occurred, the server sends a final message.
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net/http"
+	"time"
+)
+
+const (
+	timeout = 15
+)
+
+func main() {
+	log.Printf("Starting...")
+
+	checkRedirectFunc := func(req *http.Request, via []*http.Request) error {
+		log.Printf("Watch is ready!")
+		return nil
+	}
+
+	client := &http.Client{
+		Timeout:       time.Duration(timeout) * time.Second,
+		CheckRedirect: checkRedirectFunc,
+	}
+
+	id := rand.Intn(2 ^ 32 - 1)
+	body := bytes.NewBufferString("hello")
+	url := fmt.Sprintf("http://127.0.0.1:12345/watch?id=%d", id)
+	req, err := http.NewRequest("GET", url, body)
+	if err != nil {
+		log.Printf("err: %+v", err)
+		return
+	}
+	result, err := client.Do(req)
+	if err != nil {
+		log.Printf("err: %+v", err)
+		return
+	}
+	log.Printf("Event received: %+v", result)
+
+	s, err := ioutil.ReadAll(result.Body) // TODO: apparently we can stream
+	result.Body.Close()
+	log.Printf("Response: %+v", string(s))
+}

examples/longpoll/redirect-server.go

@@ -0,0 +1,56 @@
+// This is an example longpoll server. On client connection it starts a "Watch",
+// and notifies the client with a redirect when that watch is ready. This is
+// important to avoid a possible race between when the client believes the watch
+// is actually ready, and when the server actually is watching.
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"math/rand"
+	"net/http"
+	"time"
+)
+
+// you can use `wget http://127.0.0.1:12345/hello -O /dev/null`
+// or `go run client.go`
+const (
+	addr = ":12345"
+)
+
+// WatchStart kicks off the initial watch and then redirects the client to
+// notify them that we're ready. The watch operation here is simulated.
+func WatchStart(w http.ResponseWriter, req *http.Request) {
+	log.Printf("Start received...")
+	time.Sleep(time.Duration(5) * time.Second) // 5 seconds to get ready and start *our* watch ;)
+	//started := time.Now().UnixNano() // time since watch is "started"
+	log.Printf("URL: %+v", req.URL)
+
+	token := fmt.Sprintf("%d", rand.Intn(2^32-1))
+	http.Redirect(w, req, fmt.Sprintf("/ready?token=%s", token), http.StatusSeeOther) // TODO: which code should we use ?
+	log.Printf("Redirect sent!")
+}
+
+// WatchReady receives the client connection when it has been notified that the
+// watch has started, and it returns to signal that an event on the watch
+// occurred. The event operation here is simulated.
+func WatchReady(w http.ResponseWriter, req *http.Request) {
+	log.Printf("Ready received")
+	log.Printf("URL: %+v", req.URL)
+
+	//time.Sleep(time.Duration(10) * time.Second)
+	time.Sleep(time.Duration(rand.Intn(10)) * time.Second) // wait until an "event" happens
+
+	io.WriteString(w, "Event happened!\n")
+	log.Printf("Event sent")
+}
+
+func main() {
+	log.Printf("Starting...")
+	//rand.Seed(time.Now().UTC().UnixNano())
+	http.HandleFunc("/watch", WatchStart)
+	http.HandleFunc("/ready", WatchReady)
+	log.Printf("Listening on %s", addr)
+	log.Fatal(http.ListenAndServe(addr, nil))
+}

examples/nspawn1.yaml

@@ -2,6 +2,6 @@
 graph: mygraph
 resources:
   nspawn:
-  - name: mgmt-nspawn1
+  - name: nspawn1
     state: running
 edges: []

examples/nspawn2.yaml

@@ -1,7 +0,0 @@
----
-graph: mygraph
-resources:
-  nspawn:
-  - name: mgmt-nspawn2
-    state: stopped
-edges: []

examples/user1.yaml

@@ -0,0 +1,9 @@
+---
+graph: mygraph
+resources:
+  user:
+  - name: testuser
+    uid: 1002
+    gid: 100
+    state: exists
+edges: []

gapi/gapi.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 // Package gapi defines the interface that graph API generators must meet.

hcl/gapi.go

@@ -0,0 +1,155 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package hcl
+
+import (
+	"fmt"
+	"log"
+	"sync"
+
+	"github.com/purpleidea/mgmt/gapi"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/recwatch"
+)
+
+// GAPI ...
+type GAPI struct {
+	File *string
+
+	initialized   bool
+	data          gapi.Data
+	wg            sync.WaitGroup
+	closeChan     chan struct{}
+	configWatcher *recwatch.ConfigWatcher
+}
+
+// NewGAPI ...
+func NewGAPI(data gapi.Data, file *string) (*GAPI, error) {
+	if file == nil {
+		return nil, fmt.Errorf("empty file given")
+	}
+
+	obj := &GAPI{
+		File: file,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init ...
+func (obj *GAPI) Init(d gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("already initialized")
+	}
+
+	if obj.File == nil {
+		return fmt.Errorf("file cannot be nil")
+	}
+
+	obj.data = d
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	obj.configWatcher = recwatch.NewConfigWatcher()
+
+	return nil
+}
+
+// Graph ...
+func (obj *GAPI) Graph() (*pgraph.Graph, error) {
+	config, err := loadHcl(obj.File)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse graph: %s", err)
+	}
+
+	return graphFromConfig(config, obj.data)
+}
+
+// Next ...
+func (obj *GAPI) Next() chan gapi.Next {
+	ch := make(chan gapi.Next)
+	obj.wg.Add(1)
+
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch)
+		if !obj.initialized {
+			next := gapi.Next{
+				Err:  fmt.Errorf("hcl: GAPI is not initialized"),
+				Exit: true,
+			}
+			ch <- next
+			return
+		}
+		startChan := make(chan struct{}) // start signal
+		close(startChan)                 // kick it off!
+
+		watchChan, configChan := make(chan error), make(chan error)
+		if obj.data.NoConfigWatch {
+			configChan = nil
+		} else {
+			configChan = obj.configWatcher.ConfigWatch(*obj.File) // simple
+		}
+		if obj.data.NoStreamWatch {
+			watchChan = nil
+		} else {
+			watchChan = obj.data.World.ResWatch()
+		}
+
+		for {
+			var err error
+			var ok bool
+
+			select {
+			case <-startChan:
+				startChan = nil
+			case err, ok = <-watchChan:
+			case err, ok = <-configChan:
+				if !ok {
+					return
+				}
+			case <-obj.closeChan:
+				return
+			}
+
+			log.Printf("hcl: generating new graph")
+			next := gapi.Next{
+				Err: err,
+			}
+
+			select {
+			case ch <- next:
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+
+	return ch
+}
+
+// Close ...
+func (obj *GAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("hcl: GAPI is not initialized")
+	}
+
+	obj.configWatcher.Close()
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false
+	return nil
+}

hcl/parse.go

@@ -0,0 +1,387 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package hcl
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"strings"
+
+	"github.com/hashicorp/hcl"
+	"github.com/hashicorp/hcl/hcl/ast"
+	"github.com/hashicorp/hil"
+	"github.com/purpleidea/mgmt/gapi"
+	hv "github.com/purpleidea/mgmt/hil"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+)
+
+type collectorResConfig struct {
+	Kind    string
+	Pattern string
+}
+
+// Config defines the structure of the hcl config.
+type Config struct {
+	Resources []*Resource
+	Edges     []*Edge
+	Collector []collectorResConfig
+}
+
+// vertex is the data structure of a vertex.
+type vertex struct {
+	Kind string `hcl:"kind"`
+	Name string `hcl:"name"`
+}
+
+// Edge defines an edge in hcl.
+type Edge struct {
+	Name   string
+	From   vertex
+	To     vertex
+	Notify bool
+}
+
+// Resources define the state for resources.
+type Resources struct {
+	Resources []resources.Res
+}
+
+// Resource ...
+type Resource struct {
+	Name     string
+	Kind     string
+	resource resources.Res
+	Meta     resources.MetaParams
+	deps     []*Edge
+	rcv      map[string]*hv.ResourceVariable
+}
+
+type key struct {
+	kind, name string
+}
+
+func graphFromConfig(c *Config, data gapi.Data) (*pgraph.Graph, error) {
+	var graph *pgraph.Graph
+	var err error
+
+	graph, err = pgraph.NewGraph("Graph")
+	if err != nil {
+		return nil, fmt.Errorf("unable to create graph from config: %s", err)
+	}
+
+	lookup := make(map[key]pgraph.Vertex)
+
+	var keep []pgraph.Vertex
+	var resourceList []resources.Res
+
+	log.Printf("hcl: parsing %d resources", len(c.Resources))
+	for _, r := range c.Resources {
+		res := r.resource
+		kind := r.resource.GetKind()
+
+		log.Printf("hcl: resource \"%s\" \"%s\"", kind, r.Name)
+		if !strings.HasPrefix(res.GetName(), "@@") {
+			fn := func(v pgraph.Vertex) (bool, error) {
+				return resources.VtoR(v).Compare(res), nil
+			}
+			v, err := graph.VertexMatchFn(fn)
+			if err != nil {
+				return nil, fmt.Errorf("could not match vertex: %s", err)
+			}
+			if v == nil {
+				v = res
+				graph.AddVertex(v)
+			}
+			lookup[key{kind, res.GetName()}] = v
+			keep = append(keep, v)
+		} else if !data.Noop {
+			res.SetName(res.GetName()[2:])
+			res.SetKind(kind)
+			resourceList = append(resourceList, res)
+		}
+	}
+
+	// store in backend (usually etcd)
+	if err := data.World.ResExport(resourceList); err != nil {
+		return nil, fmt.Errorf("Config: Could not export resources: %v", err)
+	}
+
+	// lookup from backend (usually etcd)
+	var hostnameFilter []string // empty to get from everyone
+	kindFilter := []string{}
+	for _, t := range c.Collector {
+		kind := strings.ToLower(t.Kind)
+		kindFilter = append(kindFilter, kind)
+	}
+	// do all the graph look ups in one single step, so that if the backend
+	// database changes, we don't have a partial state of affairs...
+	if len(kindFilter) > 0 { // if kindFilter is empty, don't need to do lookups!
+		var err error
+		resourceList, err = data.World.ResCollect(hostnameFilter, kindFilter)
+		if err != nil {
+			return nil, fmt.Errorf("Config: Could not collect resources: %v", err)
+		}
+	}
+	for _, res := range resourceList {
+		matched := false
+		// see if we find a collect pattern that matches
+		for _, t := range c.Collector {
+			kind := strings.ToLower(t.Kind)
+			// use t.Kind and optionally t.Pattern to collect from storage
+			log.Printf("Collect: %v; Pattern: %v", kind, t.Pattern)
+
+			// XXX: expand to more complex pattern matching here...
+			if res.GetKind() != kind {
+				continue
+			}
+
+			if matched {
+				// we've already matched this resource, should we match again?
+				log.Printf("Config: Warning: Matching %s again!", res)
+			}
+			matched = true
+
+			// collect resources but add the noop metaparam
+			//if noop { // now done in mgmtmain
+			//	res.Meta().Noop = noop
+			//}
+
+			if t.Pattern != "" { // XXX: simplistic for now
+				res.CollectPattern(t.Pattern) // res.Dirname = t.Pattern
+			}
+
+			log.Printf("Collect: %s: collected!", res)
+
+			// XXX: similar to other resource add code:
+			// if _, exists := lookup[kind]; !exists {
+			// 	lookup[kind] = make(map[string]pgraph.Vertex)
+			// }
+
+			fn := func(v pgraph.Vertex) (bool, error) {
+				return resources.VtoR(v).Compare(res), nil
+			}
+			v, err := graph.VertexMatchFn(fn)
+			if err != nil {
+				return nil, fmt.Errorf("could not VertexMatchFn() resource: %s", err)
+			}
+			if v == nil { // no match found
+				v = res            // a standalone res can be a vertex
+				graph.AddVertex(v) // call standalone in case not part of an edge
+			}
+			lookup[key{kind, res.GetName()}] = v // used for constructing edges
+			keep = append(keep, v)               // append
+
+			//break // let's see if another resource even matches
+		}
+	}
+
+	for _, r := range c.Resources {
+		for _, e := range r.deps {
+			if _, ok := lookup[key{strings.ToLower(e.From.Kind), e.From.Name}]; !ok {
+				return nil, fmt.Errorf("can't find 'from' name")
+			}
+			if _, ok := lookup[key{strings.ToLower(e.To.Kind), e.To.Name}]; !ok {
+				return nil, fmt.Errorf("can't find 'to' name")
+			}
+			from := lookup[key{strings.ToLower(e.From.Kind), e.From.Name}]
+			to := lookup[key{strings.ToLower(e.To.Kind), e.To.Name}]
+			edge := &resources.Edge{
+				Name:   e.Name,
+				Notify: e.Notify,
+			}
+			graph.AddEdge(from, to, edge)
+		}
+
+		recv := make(map[string]*resources.Send)
+		// build Rcv's from resource variables
+		for k, v := range r.rcv {
+			send, ok := lookup[key{strings.ToLower(v.Kind), v.Name}]
+			if !ok {
+				return nil, fmt.Errorf("resource not found")
+			}
+
+			recv[strings.ToUpper(string(k[0]))+k[1:]] = &resources.Send{
+				Res: resources.VtoR(send),
+				Key: v.Field,
+			}
+
+			to := lookup[key{strings.ToLower(r.Kind), r.Name}]
+			edge := &resources.Edge{
+				Name:   v.Name,
+				Notify: true,
+			}
+			graph.AddEdge(send, to, edge)
+		}
+
+		r.resource.SetRecv(recv)
+	}
+
+	return graph, nil
+}
+
+func loadHcl(f *string) (*Config, error) {
+	if f == nil {
+		return nil, fmt.Errorf("empty file given")
+	}
+
+	data, err := ioutil.ReadFile(*f)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read file: %v", err)
+	}
+
+	file, err := hcl.ParseBytes(data)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse file: %s", err)
+	}
+
+	config := new(Config)
+
+	list, ok := file.Node.(*ast.ObjectList)
+	if !ok {
+		return nil, fmt.Errorf("unable to parse file: file does not contain root node object")
+	}
+
+	if resources := list.Filter("resource"); len(resources.Items) > 0 {
+		var err error
+		config.Resources, err = loadResourcesHcl(resources)
+		if err != nil {
+			return nil, fmt.Errorf("unable to parse: %s", err)
+		}
+	}
+
+	return config, nil
+}
+
+func loadResourcesHcl(list *ast.ObjectList) ([]*Resource, error) {
+	list = list.Children()
+	if len(list.Items) == 0 {
+		return nil, nil
+	}
+
+	var result []*Resource
+
+	for _, item := range list.Items {
+		kind := item.Keys[0].Token.Value().(string)
+		name := item.Keys[1].Token.Value().(string)
+
+		var listVal *ast.ObjectList
+		if ot, ok := item.Val.(*ast.ObjectType); ok {
+			listVal = ot.List
+		} else {
+			return nil, fmt.Errorf("module '%s': should be an object", name)
+		}
+
+		var params = resources.DefaultMetaParams
+		if o := listVal.Filter("meta"); len(o.Items) > 0 {
+			err := hcl.DecodeObject(&params, o)
+			if err != nil {
+				return nil, fmt.Errorf(
+					"Error parsing meta for %s: %s",
+					name,
+					err)
+			}
+		}
+
+		var deps []string
+		if edges := listVal.Filter("depends_on"); len(edges.Items) > 0 {
+			err := hcl.DecodeObject(&deps, edges.Items[0].Val)
+			if err != nil {
+				return nil, fmt.Errorf("unable to parse: %s", err)
+			}
+		}
+
+		var edges []*Edge
+		for _, dep := range deps {
+			vertices := strings.Split(dep, ".")
+			edges = append(edges, &Edge{
+				To: vertex{
+					Kind: kind,
+					Name: name,
+				},
+				From: vertex{
+					Kind: vertices[0],
+					Name: vertices[1],
+				},
+			})
+		}
+
+		var config map[string]interface{}
+		if err := hcl.DecodeObject(&config, item.Val); err != nil {
+			log.Printf("hcl: unable to decode body: %v", err)
+			return nil, fmt.Errorf(
+				"Error reading config for %s: %s",
+				name,
+				err)
+		}
+
+		delete(config, "meta")
+		delete(config, "depends_on")
+
+		rcv := make(map[string]*hv.ResourceVariable)
+		// parse strings for hil
+		for k, v := range config {
+			n, err := hil.Parse(v.(string))
+			if err != nil {
+				return nil, fmt.Errorf("unable to parse fields: %v", err)
+			}
+
+			variables, err := hv.ParseVariables(n)
+			if err != nil {
+				return nil, fmt.Errorf("unable to parse variables: %v", err)
+			}
+
+			for _, v := range variables {
+				val, ok := v.(*hv.ResourceVariable)
+				if !ok {
+					continue
+				}
+
+				rcv[k] = val
+			}
+		}
+
+		res, err := resources.NewNamedResource(kind, name)
+		if err != nil {
+			log.Printf("hcl: unable to parse resource: %v", err)
+			return nil, err
+		}
+
+		if err := hcl.DecodeObject(res, item.Val); err != nil {
+			log.Printf("hcl: unable to decode body: %v", err)
+			return nil, fmt.Errorf(
+				"Error reading config for %s: %s",
+				name,
+				err)
+		}
+
+		meta := res.Meta()
+		*meta = params
+
+		result = append(result, &Resource{
+			Name:     name,
+			Kind:     kind,
+			resource: res,
+			deps:     edges,
+			rcv:      rcv,
+		})
+	}
+
+	return result, nil
+}

hil/interpolate.go

@@ -0,0 +1,89 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package hil
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/hil/ast"
+)
+
+// Variable defines an interpolated variable.
+type Variable interface {
+	Key() string
+}
+
+// ResourceVariable defines a variable type used to reference fields of a resource
+// e.g.  ${file.file1.Content}
+type ResourceVariable struct {
+	Kind, Name, Field string
+}
+
+// Key returns a string representation of the variable key.
+func (r *ResourceVariable) Key() string {
+	return fmt.Sprintf("%s.%s.%s", r.Kind, r.Name, r.Field)
+}
+
+// NewInterpolatedVariable takes a variable key and return the interpolated variable
+// of the required type.
+func NewInterpolatedVariable(k string) (Variable, error) {
+	// for now resource variables are the only thing.
+	parts := strings.SplitN(k, ".", 3)
+
+	return &ResourceVariable{
+		Kind:  parts[0],
+		Name:  parts[1],
+		Field: parts[2],
+	}, nil
+}
+
+// ParseVariables will traverse a HIL tree looking for variables and returns a
+// list of them.
+func ParseVariables(tree ast.Node) ([]Variable, error) {
+	var result []Variable
+	var finalErr error
+
+	visitor := func(n ast.Node) ast.Node {
+		if finalErr != nil {
+			return n
+		}
+
+		switch nt := n.(type) {
+		case *ast.VariableAccess:
+			v, err := NewInterpolatedVariable(nt.Name)
+			if err != nil {
+				finalErr = err
+				return n
+			}
+			result = append(result, v)
+		default:
+			return n
+		}
+
+		return n
+	}
+
+	tree.Accept(visitor)
+
+	if finalErr != nil {
+		return nil, finalErr
+	}
+
+	return result, nil
+}

lib/cli.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package lib
@@ -24,6 +24,8 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/purpleidea/mgmt/bindata"
+	"github.com/purpleidea/mgmt/hcl"
 	"github.com/purpleidea/mgmt/puppet"
 	"github.com/purpleidea/mgmt/yamlgraph"
 	"github.com/purpleidea/mgmt/yamlgraph2"
@@ -89,6 +91,14 @@ func run(c *cli.Context) error {
 			PuppetConf:  c.String("puppet-conf"),
 		}
 	}
+	if h := c.String("hcl"); c.IsSet("hcl") {
+		if obj.GAPI != nil {
+			return fmt.Errorf("can't combine hcl GAPI with existing GAPI")
+		}
+		obj.GAPI = &hcl.GAPI{
+			File: &h,
+		}
+	}
 	obj.Remotes = c.StringSlice("remote") // FIXME: GAPI-ify somehow?
 
 	obj.NoWatch = c.Bool("no-watch")
@@ -105,6 +115,8 @@ func run(c *cli.Context) error {
 	obj.Seeds = c.StringSlice("seeds")
 	obj.ClientURLs = c.StringSlice("client-urls")
 	obj.ServerURLs = c.StringSlice("server-urls")
+	obj.AdvertiseClientURLs = c.StringSlice("advertise-client-urls")
+	obj.AdvertiseServerURLs = c.StringSlice("advertise-server-urls")
 	obj.IdealClusterSize = c.Int("ideal-cluster-size")
 	obj.NoServer = c.Bool("no-server")
 
@@ -177,7 +189,32 @@ func CLI(program, version string, flags Flags) error {
 	app.Metadata = map[string]interface{}{ // additional flags
 		"flags": flags,
 	}
-	//app.Action = ... // without a default action, help runs
+
+	// if no app.Command is specified
+	app.Action = func(c *cli.Context) error {
+		// print the license
+		if c.Bool("license") {
+			license, err := bindata.Asset("../COPYING") // use go-bindata to get the bytes
+			if err != nil {
+				return err
+			}
+
+			fmt.Printf("%s", license)
+			return nil
+		}
+
+		// print help if no flags are set
+		cli.ShowAppHelp(c)
+		return nil
+	}
+
+	// global flags
+	app.Flags = []cli.Flag{
+		cli.BoolFlag{
+			Name:  "license",
+			Usage: "prints the software license",
+		},
+	}
 
 	app.Commands = []cli.Command{
 		{
@@ -222,6 +259,11 @@ func CLI(program, version string, flags Flags) error {
 					Value: "",
 					Usage: "yaml graph definition to run (parser v2)",
 				},
+				cli.StringFlag{
+					Name:  "hcl",
+					Value: "",
+					Usage: "hcl graph definition to run",
+				},
 				cli.StringFlag{
 					Name:  "puppet, p",
 					Value: "",
@@ -304,6 +346,20 @@ func CLI(program, version string, flags Flags) error {
 					Usage:  "list of URLs to listen on for server (peer) traffic",
 					EnvVar: "MGMT_SERVER_URLS",
 				},
+				// port 2379 and 4001 are common
+				cli.StringSliceFlag{
+					Name:   "advertise-client-urls",
+					Value:  &cli.StringSlice{},
+					Usage:  "list of URLs to listen on for client traffic",
+					EnvVar: "MGMT_ADVERTISE_CLIENT_URLS",
+				},
+				// port 2380 and 7001 are common
+				cli.StringSliceFlag{
+					Name:   "advertise-server-urls, advertise-peer-urls",
+					Value:  &cli.StringSlice{},
+					Usage:  "list of URLs to listen on for server (peer) traffic",
+					EnvVar: "MGMT_ADVERTISE_SERVER_URLS",
+				},
 				cli.IntFlag{
 					Name:   "ideal-cluster-size",
 					Value:  -1,

lib/main.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package lib
@@ -79,6 +79,8 @@ type Main struct {
 	Seeds               []string // default etc client endpoint
 	ClientURLs          []string // list of URLs to listen on for client traffic
 	ServerURLs          []string // list of URLs to listen on for server (peer) traffic
+	AdvertiseClientURLs []string // list of URLs to advertise for client traffic
+	AdvertiseServerURLs []string // list of URLs to advertise for server (peer) traffic
 	IdealClusterSize    int      // ideal number of server peers in cluster; only read by initial server
 	NoServer            bool     // do not let other servers peer with me
 
@@ -91,6 +93,8 @@ type Main struct {
 	seeds               etcdtypes.URLs // processed seeds value
 	clientURLs          etcdtypes.URLs // processed client urls value
 	serverURLs          etcdtypes.URLs // processed server urls value
+	advertiseClientURLs etcdtypes.URLs // processed advertise client urls value
+	advertiseServerURLs etcdtypes.URLs // processed advertise server urls value
 	idealClusterSize    uint16         // processed ideal cluster size value
 
 	NoPgp       bool    // disallow pgp functionality
@@ -173,6 +177,18 @@ func (obj *Main) Init() error {
 	if err != nil && len(obj.ServerURLs) > 0 {
 		return fmt.Errorf("the ServerURLs didn't parse correctly")
 	}
+	obj.advertiseClientURLs, err = etcdtypes.NewURLs(
+		util.FlattenListWithSplit(obj.AdvertiseClientURLs, []string{",", ";", " "}),
+	)
+	if err != nil && len(obj.AdvertiseClientURLs) > 0 {
+		return fmt.Errorf("the AdvertiseClientURLs didn't parse correctly")
+	}
+	obj.advertiseServerURLs, err = etcdtypes.NewURLs(
+		util.FlattenListWithSplit(obj.AdvertiseServerURLs, []string{",", ";", " "}),
+	)
+	if err != nil && len(obj.AdvertiseServerURLs) > 0 {
+		return fmt.Errorf("the AdvertiseServerURLs didn't parse correctly")
+	}
 
 	obj.exit = make(chan error)
 	return nil
@@ -253,6 +269,10 @@ func (obj *Main) Run() error {
 		if err := prom.Start(); err != nil {
 			return errwrap.Wrapf(err, "can't start initiate Prometheus instance")
 		}
+
+		if err := prom.InitKindMetrics(resources.RegisteredResourcesNames()); err != nil {
+			return errwrap.Wrapf(err, "can't initialize kind-specific prometheus metrics")
+		}
 	}
 
 	if !obj.NoPgp {
@@ -330,6 +350,8 @@ func (obj *Main) Run() error {
 		obj.seeds,
 		obj.clientURLs,
 		obj.serverURLs,
+		obj.advertiseClientURLs,
+		obj.advertiseServerURLs,
 		obj.NoServer,
 		obj.idealClusterSize,
 		etcd.Flags{
@@ -524,6 +546,8 @@ func (obj *Main) Run() error {
 				continue
 			}
 
+			//savedGraph := oldGraph.Copy() // save a copy for errors
+
 			// TODO: should we call each Res.Setup() here instead?
 
 			// add autoedges; modifies the graph only if no error
@@ -537,12 +561,20 @@ func (obj *Main) Run() error {
 				continue
 			}
 
-			graph.Update(oldGraph) // copy in structure of new graph
+			// at this point, any time we error after a destructive
+			// modification of the graph we need to restore the old
+			// graph that was previously running, eg:
+			//
+			//	oldGraph = savedGraph.Copy()
+			//
+			// which we are (luckily) able to avoid testing for now
 
-			resources.AutoGroup(graph.Graph, &resources.NonReachabilityGrouper{}) // run autogroup; modifies the graph
+			resources.AutoGroup(oldGraph, &resources.NonReachabilityGrouper{}) // run autogroup; modifies the graph
 			// TODO: do we want to do a transitive reduction?
 			// FIXME: run a type checker that verifies all the send->recv relationships
 
+			graph.Update(oldGraph) // copy in structure of new graph
+
 			// Call this here because at this point the graph does
 			// not know anything about the prometheus instance.
 			if err := prom.UpdatePgraphStartTime(); err != nil {
@@ -619,6 +651,14 @@ func (obj *Main) Run() error {
 	// TODO: is there any benefit to running the remotes above in the loop?
 	// wait for etcd to be running before we remote in, which we do above!
 	go remotes.Run()
+	// wait for remotes to be ready before continuing...
+	select {
+	case <-remotes.Ready():
+		log.Printf("Main: Remotes: Run: Ready!")
+		// pass
+		//case <-time.After( ? * time.Second):
+		//	obj.Exit(fmt.Errorf("Main: Remotes: Run timeout"))
+	}
 
 	if obj.GAPI == nil {
 		converger.Start() // better start this for empty graphs

main.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package main

misc/make-deps.sh

@@ -79,6 +79,7 @@ if [[ $ret != 0 ]]; then
 	go get golang.org/x/tools/cmd/vet      # add in `go vet` for travis
 fi
 go get golang.org/x/tools/cmd/stringer			# for automatic stringer-ing
+go get github.com/jteeuwen/go-bindata/go-bindata	# for compiling in non golang files
 go get github.com/golang/lint/golint			# for `golint`-ing
-go get github.com/alecthomas/gometalinter && gometalinter --install	# bonus
+go get -u gopkg.in/alecthomas/gometalinter.v1 && mv "$(dirname $(which gometalinter.v1))/gometalinter.v1" "$(dirname $(which gometalinter.v1))/gometalinter" && gometalinter --install	# bonus
 cd "$XPWD" >/dev/null

pgp/pgp.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package pgp

pgraph/graphsync.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package pgraph
@@ -23,12 +23,25 @@ import (
 	errwrap "github.com/pkg/errors"
 )
 
+func strVertexCmpFn(v1, v2 Vertex) (bool, error) {
+	if v1.String() == "" || v2.String() == "" {
+		return false, fmt.Errorf("empty vertex")
+	}
+	return v1.String() == v2.String(), nil
+}
+
+func strEdgeCmpFn(e1, e2 Edge) (bool, error) {
+	if e1.String() == "" || e2.String() == "" {
+		return false, fmt.Errorf("empty edge")
+	}
+	return e1.String() == e2.String(), nil
+}
+
 // GraphSync updates the Graph so that it matches the newGraph. It leaves
 // identical elements alone so that they don't need to be refreshed.
 // It tries to mutate existing elements into new ones, if they support this.
 // This updates the Graph on success only.
 // FIXME: should we do this with copies of the vertex resources?
-// FIXME: add test cases
 func (obj *Graph) GraphSync(newGraph *Graph, vertexCmpFn func(Vertex, Vertex) (bool, error), vertexAddFn func(Vertex) error, vertexRemoveFn func(Vertex) error, edgeCmpFn func(Edge, Edge) (bool, error)) error {
 
 	oldGraph := obj.Copy() // work on a copy of the old graph
@@ -41,6 +54,19 @@ func (obj *Graph) GraphSync(newGraph *Graph, vertexCmpFn func(Vertex, Vertex) (b
 	}
 	oldGraph.SetName(newGraph.GetName()) // overwrite the name
 
+	if vertexCmpFn == nil {
+		vertexCmpFn = strVertexCmpFn // use simple string cmp version
+	}
+	if vertexAddFn == nil {
+		vertexAddFn = func(Vertex) error { return nil } // noop
+	}
+	if vertexRemoveFn == nil {
+		vertexRemoveFn = func(Vertex) error { return nil } // noop
+	}
+	if edgeCmpFn == nil {
+		edgeCmpFn = strEdgeCmpFn // use simple string cmp version
+	}
+
 	var lookup = make(map[Vertex]Vertex)
 	var vertexKeep []Vertex // list of vertices which are the same in new graph
 	var edgeKeep []Edge     // list of vertices which are the same in new graph

pgraph/graphsync_test.go

@@ -0,0 +1,92 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgraph
+
+import (
+	"testing"
+)
+
+func TestGraphSync1(t *testing.T) {
+	g := &Graph{}
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+
+	e1 := NE("e1")
+	e2 := NE("e2")
+	g.AddEdge(v1, v3, e1)
+	g.AddEdge(v2, v3, e2)
+
+	// new graph
+	newGraph := &Graph{}
+	v4 := NV("v4")
+	v5 := NV("v5")
+	e3 := NE("e3")
+	newGraph.AddEdge(v4, v5, e3)
+
+	err := g.GraphSync(newGraph, nil, nil, nil, nil)
+	if err != nil {
+		t.Errorf("GraphSync failed: %v", err)
+		return
+	}
+
+	// g should change and become the same
+	if s := runGraphCmp(t, g, newGraph); s != "" {
+		t.Errorf("%s", s)
+	}
+}
+
+func TestGraphSync2(t *testing.T) {
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+	v5 := NV("v5")
+	e1 := NE("e1")
+	e2 := NE("e2")
+	e3 := NE("e3")
+
+	g := &Graph{}
+	g.AddEdge(v1, v3, e1)
+	g.AddEdge(v2, v3, e2)
+
+	// new graph
+	newGraph := &Graph{}
+	newGraph.AddEdge(v1, v3, e1)
+	newGraph.AddEdge(v2, v3, e2)
+	newGraph.AddEdge(v4, v5, e3)
+	//newGraph.AddEdge(v3, v4, NE("v3,v4"))
+	//newGraph.AddEdge(v3, v5, NE("v3,v5"))
+
+	// graphs should differ!
+	if runGraphCmp(t, g, newGraph) == "" {
+		t.Errorf("graphs should differ initially")
+		return
+	}
+
+	err := g.GraphSync(newGraph, strVertexCmpFn, vertexAddFn, vertexRemoveFn, strEdgeCmpFn)
+	if err != nil {
+		t.Errorf("GraphSync failed: %v", err)
+		return
+	}
+
+	// g should change and become the same
+	if s := runGraphCmp(t, g, newGraph); s != "" {
+		t.Errorf("%s", s)
+	}
+}

pgraph/graphviz.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package pgraph // TODO: this should be a subpackage

pgraph/pgraph.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 // Package pgraph represents the internal "pointer graph" that we use.
@@ -142,7 +142,6 @@ func (g *Graph) AddEdge(v1, v2 Vertex, e Edge) {
 }
 
 // DeleteEdge deletes a particular edge from the graph.
-// FIXME: add test cases
 func (g *Graph) DeleteEdge(e Edge) {
 	for v1 := range g.adjacency {
 		for v2, edge := range g.adjacency[v1] {

pgraph/pgraph_test.go

@@ -1,60 +1,28 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package pgraph
 
 import (
-	"fmt"
 	"reflect"
 	"testing"
 )
 
-// vertex is a test struct to test the library.
+func TestCount1(t *testing.T) {
-type vertex struct {
-	name string
-}
-
-// String is a required method of the Vertex interface that we must fulfill.
-func (v *vertex) String() string {
-	return v.name
-}
-
-// NV is a helper function to make testing easier. It creates a new noop vertex.
-func NV(s string) Vertex {
-	return &vertex{s}
-}
-
-// edge is a test struct to test the library.
-type edge struct {
-	name string
-}
-
-// String is a required method of the Edge interface that we must fulfill.
-func (e *edge) String() string {
-	return e.name
-}
-
-// NE is a helper function to make testing easier. It creates a new noop edge.
-func NE(s string) Edge {
-	return &edge{s}
-}
-
-func TestPgraphT1(t *testing.T) {
-
 	G := &Graph{}
 
 	if i := G.NumVertices(); i != 0 {
@@ -79,8 +47,7 @@ func TestPgraphT1(t *testing.T) {
 	}
 }
 
-func TestPgraphT2(t *testing.T) {
+func TestAddVertex1(t *testing.T) {
-
 	G := &Graph{Name: "g2"}
 	v1 := NV("v1")
 	v2 := NV("v2")
@@ -106,8 +73,7 @@ func TestPgraphT2(t *testing.T) {
 	}
 }
 
-func TestPgraphT3(t *testing.T) {
+func TestDFS1(t *testing.T) {
-
 	G, _ := NewGraph("g3")
 	v1 := NV("v1")
 	v2 := NV("v2")
@@ -147,8 +113,7 @@ func TestPgraphT3(t *testing.T) {
 	}
 }
 
-func TestPgraphT4(t *testing.T) {
+func TestDFS2(t *testing.T) {
-
 	G, _ := NewGraph("g4")
 	v1 := NV("v1")
 	v2 := NV("v2")
@@ -170,7 +135,7 @@ func TestPgraphT4(t *testing.T) {
 	}
 }
 
-func TestPgraphT5(t *testing.T) {
+func TestFilterGraph1(t *testing.T) {
 	G, _ := NewGraph("g5")
 	v1 := NV("v1")
 	v2 := NV("v2")
@@ -203,7 +168,7 @@ func TestPgraphT5(t *testing.T) {
 	}
 }
 
-func TestPgraphT6(t *testing.T) {
+func TestDisconnectedGraphs1(t *testing.T) {
 	G, _ := NewGraph("g6")
 	v1 := NV("v1")
 	v2 := NV("v2")
@@ -235,8 +200,7 @@ func TestPgraphT6(t *testing.T) {
 	}
 }
 
-func TestPgraphT7(t *testing.T) {
+func TestDeleteVertex1(t *testing.T) {
-
 	G, _ := NewGraph("g7")
 	v1 := NV("v1")
 	v2 := NV("v2")
@@ -277,8 +241,17 @@ func TestPgraphT7(t *testing.T) {
 	}
 }
 
-func TestPgraphT8(t *testing.T) {
+func TestDeleteVertex2(t *testing.T) {
+	G := &Graph{}
+	v1 := NV("v1")
+	G.DeleteVertex(v1) // check this doesn't panic
 
+	if i := G.NumVertices(); i != 0 {
+		t.Errorf("should have 0 vertices instead of: %d", i)
+	}
+}
+
+func TestVertexContains1(t *testing.T) {
 	v1 := NV("v1")
 	v2 := NV("v2")
 	v3 := NV("v3")
@@ -306,8 +279,7 @@ func TestPgraphT8(t *testing.T) {
 	}
 }
 
-func TestPgraphT9(t *testing.T) {
+func TestTopoSort1(t *testing.T) {
-
 	G, _ := NewGraph("g9")
 	v1 := NV("v1")
 	v2 := NV("v2")
@@ -382,8 +354,7 @@ func TestPgraphT9(t *testing.T) {
 	}
 }
 
-func TestPgraphT10(t *testing.T) {
+func TestTopoSort2(t *testing.T) {
-
 	G, _ := NewGraph("g10")
 	v1 := NV("v1")
 	v2 := NV("v2")
@@ -410,7 +381,7 @@ func TestPgraphT10(t *testing.T) {
 }
 
 // empty
-func TestPgraphReachability0(t *testing.T) {
+func TestReachability0(t *testing.T) {
 	{
 		G, _ := NewGraph("g")
 		result := G.Reachability(nil, nil)
@@ -474,7 +445,7 @@ func TestPgraphReachability0(t *testing.T) {
 }
 
 // simple linear path
-func TestPgraphReachability1(t *testing.T) {
+func TestReachability1(t *testing.T) {
 	G, _ := NewGraph("g")
 	v1 := NV("v1")
 	v2 := NV("v2")
@@ -508,7 +479,7 @@ func TestPgraphReachability1(t *testing.T) {
 }
 
 // pick one of two correct paths
-func TestPgraphReachability2(t *testing.T) {
+func TestReachability2(t *testing.T) {
 	G, _ := NewGraph("g")
 	v1 := NV("v1")
 	v2 := NV("v2")
@@ -545,7 +516,7 @@ func TestPgraphReachability2(t *testing.T) {
 }
 
 // pick shortest path
-func TestPgraphReachability3(t *testing.T) {
+func TestReachability3(t *testing.T) {
 	G, _ := NewGraph("g")
 	v1 := NV("v1")
 	v2 := NV("v2")
@@ -580,7 +551,7 @@ func TestPgraphReachability3(t *testing.T) {
 }
 
 // direct path
-func TestPgraphReachability4(t *testing.T) {
+func TestReachability4(t *testing.T) {
 	G, _ := NewGraph("g")
 	v1 := NV("v1")
 	v2 := NV("v2")
@@ -614,7 +585,7 @@ func TestPgraphReachability4(t *testing.T) {
 	}
 }
 
-func TestPgraphT11(t *testing.T) {
+func TestReverse1(t *testing.T) {
 	v1 := NV("v1")
 	v2 := NV("v2")
 	v3 := NV("v3")
@@ -639,7 +610,7 @@ func TestPgraphT11(t *testing.T) {
 	}
 }
 
-func TestPgraphCopy1(t *testing.T) {
+func TestCopy1(t *testing.T) {
 	g1 := &Graph{}
 	g2 := g1.Copy() // check this doesn't panic
 	if !reflect.DeepEqual(g1.String(), g2.String()) {
@@ -647,31 +618,7 @@ func TestPgraphCopy1(t *testing.T) {
 	}
 }
 
-func TestPgraphDelete1(t *testing.T) {
+func TestGraphCmp1(t *testing.T) {
-	G := &Graph{}
-	v1 := NV("v1")
-	G.DeleteVertex(v1) // check this doesn't panic
-
-	if i := G.NumVertices(); i != 0 {
-		t.Errorf("should have 0 vertices instead of: %d", i)
-	}
-}
-
-func vertexCmpFn(v1, v2 Vertex) (bool, error) {
-	if v1.String() == "" || v2.String() == "" {
-		return false, fmt.Errorf("oops, empty vertex")
-	}
-	return v1.String() == v2.String(), nil
-}
-
-func edgeCmpFn(e1, e2 Edge) (bool, error) {
-	if e1.String() == "" || e2.String() == "" {
-		return false, fmt.Errorf("oops, empty edge")
-	}
-	return e1.String() == e2.String(), nil
-}
-
-func TestPgraphGraphCmp1(t *testing.T) {
 	g1 := &Graph{}
 	g2 := &Graph{}
 	g3 := &Graph{}
@@ -679,20 +626,20 @@ func TestPgraphGraphCmp1(t *testing.T) {
 	g4 := &Graph{}
 	g4.AddVertex(NV("v2"))
 
-	if err := g1.GraphCmp(g2, vertexCmpFn, edgeCmpFn); err != nil {
+	if err := g1.GraphCmp(g2, strVertexCmpFn, strEdgeCmpFn); err != nil {
 		t.Errorf("should have no error during GraphCmp, but got: %v", err)
 	}
 
-	if err := g1.GraphCmp(g3, vertexCmpFn, edgeCmpFn); err == nil {
+	if err := g1.GraphCmp(g3, strVertexCmpFn, strEdgeCmpFn); err == nil {
 		t.Errorf("should have error during GraphCmp, but got nil")
 	}
 
-	if err := g3.GraphCmp(g4, vertexCmpFn, edgeCmpFn); err == nil {
+	if err := g3.GraphCmp(g4, strVertexCmpFn, strEdgeCmpFn); err == nil {
 		t.Errorf("should have error during GraphCmp, but got nil")
 	}
 }
 
-func TestPgraphSort0(t *testing.T) {
+func TestSort0(t *testing.T) {
 	vs := []Vertex{}
 	s := Sort(vs)
 
@@ -710,7 +657,7 @@ func TestPgraphSort0(t *testing.T) {
 	}
 }
 
-func TestPgraphSort1(t *testing.T) {
+func TestSort1(t *testing.T) {
 	v1 := NV("v1")
 	v2 := NV("v2")
 	v3 := NV("v3")
@@ -739,3 +686,84 @@ func TestPgraphSort1(t *testing.T) {
 		t.Errorf(str)
 	}
 }
+
+func TestDeleteEdge1(t *testing.T) {
+	g, _ := NewGraph("g")
+
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+
+	e1 := NE("e1")
+	e2 := NE("e2")
+	e3 := NE("e3")
+	e4 := NE("e4")
+	e5 := NE("e5")
+	e6 := NE("e6")
+
+	g.AddEdge(v1, v2, e1)
+	g.AddEdge(v2, v3, e2)
+	g.AddEdge(v1, v3, e3)
+	g.AddEdge(v2, v1, e4)
+	g.AddEdge(v3, v2, e5)
+	g.AddEdge(v3, v1, e6)
+
+	g.DeleteEdge(e1)
+	g.DeleteEdge(e2)
+	g.DeleteEdge(e3)
+	g.DeleteEdge(e3)
+
+	if g.NumEdges() != 3 {
+		t.Errorf("expected number of edges: 3, instead of: %d", g.NumEdges())
+	}
+}
+
+func TestDeleteEdge2(t *testing.T) {
+	g, _ := NewGraph("g")
+
+	v1 := NV("v1")
+	v2 := NV("v2")
+	v3 := NV("v3")
+	v4 := NV("v4")
+
+	e1 := NE("e1")
+	e2 := NE("e2")
+	e3 := NE("e3")
+	e4 := NE("e4")
+	e5 := NE("e5")
+	e6 := NE("e6")
+	e7 := NE("e7")
+
+	g.AddEdge(v1, v2, e1)
+	g.AddEdge(v1, v2, e2)
+	g.AddEdge(v1, v3, e3)
+	g.AddEdge(v1, v4, e4)
+	g.AddEdge(v2, v1, e5)
+	g.AddEdge(v3, v1, e6)
+	g.AddEdge(v4, v1, e7)
+
+	g.DeleteEdge(e1)
+	g.DeleteEdge(e2)
+	g.DeleteEdge(e3)
+	g.DeleteEdge(e5)
+	g.DeleteEdge(e6)
+
+	ie := g.IncomingGraphEdges(v1)
+	oe := g.OutgoingGraphEdges(v1)
+
+	if !reflect.DeepEqual(ie, []Edge{e7}) {
+		res := ""
+		for _, e := range ie {
+			res += e.String() + " "
+		}
+		t.Errorf("expected incoming graph edges for vertex v1: e7, instead of: %s", res)
+	}
+
+	if !reflect.DeepEqual(oe, []Edge{e4}) {
+		res := ""
+		for _, e := range oe {
+			res += e.String() + " "
+		}
+		t.Errorf("expected outgoing graph edges for vertex v1: e4, instead of: %s", res)
+	}
+}

pgraph/subgraph.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package pgraph

pgraph/subgraph_test.go

@@ -1,60 +1,27 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package pgraph
 
 import (
-	"fmt"
 	"testing"
 )
 
-// TODO: unify with the other function like this...
+func TestAddEdgeGraph1(t *testing.T) {
-// TODO: where should we put our test helpers?
-func runGraphCmp(t *testing.T, g1, g2 *Graph) {
-	err := g1.GraphCmp(g2, vertexCmpFn, edgeCmpFn)
-	if err != nil {
-		t.Logf("  actual (g1): %v%v", g1, fullPrint(g1))
-		t.Logf("expected (g2): %v%v", g2, fullPrint(g2))
-		t.Logf("Cmp error:")
-		t.Errorf("%v", err)
-	}
-}
-
-// TODO: unify with the other function like this...
-func fullPrint(g *Graph) (str string) {
-	str += "\n"
-	for v := range g.Adjacency() {
-		str += fmt.Sprintf("* v: %s\n", v)
-	}
-	for v1 := range g.Adjacency() {
-		for v2, e := range g.Adjacency()[v1] {
-			str += fmt.Sprintf("* e: %s -> %s # %s\n", v1, v2, e)
-		}
-	}
-	return
-}
-
-// edgeGenFn generates unique edges for each vertex pair, assuming unique
-// vertices.
-func edgeGenFn(v1, v2 Vertex) Edge {
-	return NE(fmt.Sprintf("%s,%s", v1, v2))
-}
-
-func TestPgraphAddEdgeGraph1(t *testing.T) {
 	v1 := NV("v1")
 	v2 := NV("v2")
 	v3 := NV("v3")
@@ -82,10 +49,12 @@ func TestPgraphAddEdgeGraph1(t *testing.T) {
 	//expected.AddEdge(v3, v4, NE("v3,v4"))
 	//expected.AddEdge(v3, v5, NE("v3,v5"))
 
-	runGraphCmp(t, g, expected)
+	if s := runGraphCmp(t, g, expected); s != "" {
+		t.Errorf("%s", s)
+	}
 }
 
-func TestPgraphAddEdgeVertexGraph1(t *testing.T) {
+func TestAddEdgeVertexGraph1(t *testing.T) {
 	v1 := NV("v1")
 	v2 := NV("v2")
 	v3 := NV("v3")
@@ -113,10 +82,12 @@ func TestPgraphAddEdgeVertexGraph1(t *testing.T) {
 	expected.AddEdge(v3, v4, NE("v3,v4"))
 	expected.AddEdge(v3, v5, NE("v3,v5"))
 
-	runGraphCmp(t, g, expected)
+	if s := runGraphCmp(t, g, expected); s != "" {
+		t.Errorf("%s", s)
+	}
 }
 
-func TestPgraphAddEdgeGraphVertex1(t *testing.T) {
+func TestAddEdgeGraphVertex1(t *testing.T) {
 	v1 := NV("v1")
 	v2 := NV("v2")
 	v3 := NV("v3")
@@ -144,10 +115,12 @@ func TestPgraphAddEdgeGraphVertex1(t *testing.T) {
 	expected.AddEdge(v4, v3, NE("v4,v3"))
 	expected.AddEdge(v5, v3, NE("v5,v3"))
 
-	runGraphCmp(t, g, expected)
+	if s := runGraphCmp(t, g, expected); s != "" {
+		t.Errorf("%s", s)
+	}
 }
 
-func TestPgraphAddEdgeVertexGraphLight1(t *testing.T) {
+func TestAddEdgeVertexGraphLight1(t *testing.T) {
 	v1 := NV("v1")
 	v2 := NV("v2")
 	v3 := NV("v3")
@@ -175,10 +148,12 @@ func TestPgraphAddEdgeVertexGraphLight1(t *testing.T) {
 	expected.AddEdge(v3, v4, NE("v3,v4"))
 	//expected.AddEdge(v3, v5, NE("v3,v5")) // not needed with light
 
-	runGraphCmp(t, g, expected)
+	if s := runGraphCmp(t, g, expected); s != "" {
+		t.Errorf("%s", s)
+	}
 }
 
-func TestPgraphAddEdgeGraphVertexLight1(t *testing.T) {
+func TestAddEdgeGraphVertexLight1(t *testing.T) {
 	v1 := NV("v1")
 	v2 := NV("v2")
 	v3 := NV("v3")
@@ -206,5 +181,7 @@ func TestPgraphAddEdgeGraphVertexLight1(t *testing.T) {
 	//expected.AddEdge(v4, v3, NE("v4,v3")) // not needed with light
 	expected.AddEdge(v5, v3, NE("v5,v3"))
 
-	runGraphCmp(t, g, expected)
+	if s := runGraphCmp(t, g, expected); s != "" {
+		t.Errorf("%s", s)
+	}
 }

pgraph/util_test.go

@@ -0,0 +1,93 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package pgraph
+
+import (
+	"fmt"
+	"testing"
+)
+
+// vertex is a test struct to test the library.
+type vertex struct {
+	name string
+}
+
+// String is a required method of the Vertex interface that we must fulfill.
+func (v *vertex) String() string {
+	return v.name
+}
+
+// NV is a helper function to make testing easier. It creates a new noop vertex.
+func NV(s string) Vertex {
+	return &vertex{s}
+}
+
+// edge is a test struct to test the library.
+type edge struct {
+	name string
+}
+
+// String is a required method of the Edge interface that we must fulfill.
+func (e *edge) String() string {
+	return e.name
+}
+
+// NE is a helper function to make testing easier. It creates a new noop edge.
+func NE(s string) Edge {
+	return &edge{s}
+}
+
+// edgeGenFn generates unique edges for each vertex pair, assuming unique
+// vertices.
+func edgeGenFn(v1, v2 Vertex) Edge {
+	return NE(fmt.Sprintf("%s,%s", v1, v2))
+}
+
+func vertexAddFn(v Vertex) error {
+	return nil
+}
+
+func vertexRemoveFn(v Vertex) error {
+	return nil
+}
+
+func runGraphCmp(t *testing.T, g1, g2 *Graph) string {
+	err := g1.GraphCmp(g2, strVertexCmpFn, strEdgeCmpFn)
+	if err != nil {
+		str := ""
+		str += fmt.Sprintf("  actual (g1): %v%s", g1, fullPrint(g1))
+		str += fmt.Sprintf("expected (g2): %v%s", g2, fullPrint(g2))
+		str += fmt.Sprintf("cmp error:")
+		str += fmt.Sprintf("%v", err)
+		return str
+	}
+	return ""
+}
+
+func fullPrint(g *Graph) (str string) {
+	str += "\n"
+	for v := range g.Adjacency() {
+		str += fmt.Sprintf("* v: %s\n", v)
+	}
+	for v1 := range g.Adjacency() {
+		for v2, e := range g.Adjacency()[v1] {
+			str += fmt.Sprintf("* e: %s -> %s # %s\n", v1, v2, e)
+		}
+	}
+	return
+}

prometheus/prometheus.go

@@ -1,22 +1,22 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 // Package prometheus provides functions that are useful to control and manage
-// the build-in prometheus instance.
+// the built-in prometheus instance.
 package prometheus
 
 import (
@@ -149,6 +149,40 @@ func (obj *Prometheus) Stop() error {
 	return nil
 }
 
+// InitKindMetrics initialized prometheus counters. For each kind of
+// resource checkApply counters are initialized with all the possible value.
+func (obj *Prometheus) InitKindMetrics(kinds []string) error {
+	if obj == nil {
+		return nil // happens when mgmt is launched without --prometheus
+	}
+	bools := []bool{true, false}
+	for _, kind := range kinds {
+		for _, apply := range bools {
+			for _, eventful := range bools {
+				for _, errorful := range bools {
+					labels := prometheus.Labels{
+						"kind":     kind,
+						"apply":    strconv.FormatBool(apply),
+						"eventful": strconv.FormatBool(eventful),
+						"errorful": strconv.FormatBool(errorful),
+					}
+					obj.checkApplyTotal.With(labels)
+				}
+			}
+		}
+
+		obj.managedResources.With(prometheus.Labels{"kind": kind})
+
+		failures := []string{"soft", "hard"}
+		for _, f := range failures {
+			failLabels := prometheus.Labels{"kind": kind, "failure": f}
+			obj.failedResourcesTotal.With(failLabels)
+			obj.failedResources.With(failLabels)
+		}
+	}
+	return nil
+}
+
 // UpdateCheckApplyTotal refreshes the failing gauge by parsing the internal
 // state map.
 func (obj *Prometheus) UpdateCheckApplyTotal(kind string, apply, eventful, errorful bool) error {

prometheus/prometheus_test.go

@@ -0,0 +1,75 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prometheus
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// TestInitKindMetrics tests that we are initializing the Prometheus
+// metrics correctly for all kind of resources.
+func TestInitKindMetrics(t *testing.T) {
+	var prom Prometheus
+	prom.Init()
+	prom.InitKindMetrics([]string{"file", "exec"})
+
+	// Get a list of metrics collected by Prometheus.
+	// This is the only way to get Prometheus metrics
+	// without implicitly creating them.
+	gatherer := prometheus.DefaultGatherer
+	metrics, err := gatherer.Gather()
+	if err != nil {
+		t.Errorf("Error while gathering metrics: %s", err)
+		return
+	}
+
+	// expectedMetrics is a map: keys are metrics name and
+	// values are expected and actual count of metrics with
+	// that name.
+	expectedMetrics := map[string][2]int{
+		"mgmt_checkapply_total": {
+			16, 0,
+		},
+		"mgmt_failures_total": {
+			4, 0,
+		},
+		"mgmt_failures": {
+			4, 0,
+		},
+		"mgmt_resources": {
+			2, 0,
+		},
+	}
+
+	for _, metric := range metrics {
+		for name, count := range expectedMetrics {
+			if *metric.Name == name {
+				value := len(metric.Metric)
+				expectedMetrics[name] = [2]int{count[0], value}
+			}
+		}
+	}
+
+	for name, count := range expectedMetrics {
+		if count[1] != count[0] {
+			t.Errorf("%s: Expected %d metrics, got %d metrics", name, count[0], count[1])
+		}
+	}
+}

puppet/gapi.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package puppet

puppet/puppet.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 // Package puppet provides the integration entrypoint for the puppet language.
@@ -89,20 +89,21 @@ func runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
 // ParseConfigFromPuppet takes a special puppet param string and config and
 // returns the graph configuration structure.
 func ParseConfigFromPuppet(puppetParam, puppetConf string) *yamlgraph.GraphConfig {
-	var puppetConfArg string
+	var args []string
-	if puppetConf != "" {
+	if puppetParam == "agent" {
-		puppetConfArg = "--config=" + puppetConf
+		args = []string{"mgmtgraph", "print"}
+	} else if strings.HasSuffix(puppetParam, ".pp") {
+		args = []string{"mgmtgraph", "print", "--manifest", puppetParam}
+	} else {
+		args = []string{"mgmtgraph", "print", "--code", puppetParam}
 	}
 
-	var cmd *exec.Cmd
+	if puppetConf != "" {
-	if puppetParam == "agent" {
+		args = append(args, "--config="+puppetConf)
-		cmd = exec.Command("puppet", "mgmtgraph", "print", puppetConfArg)
-	} else if strings.HasSuffix(puppetParam, ".pp") {
-		cmd = exec.Command("puppet", "mgmtgraph", "print", puppetConfArg, "--manifest", puppetParam)
-	} else {
-		cmd = exec.Command("puppet", "mgmtgraph", "print", puppetConfArg, "--code", puppetParam)
 	}
 
+	cmd := exec.Command("puppet", args...)
+
 	log.Println("Puppet: launching translator")
 
 	var config yamlgraph.GraphConfig

recwatch/configwatch.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package recwatch

recwatch/flags.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package recwatch

recwatch/recwatch.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 // Package recwatch provides recursive file watching events via fsnotify.

remote/remote.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 // Package remote provides the remoting facilities for agentless execution.
@@ -692,7 +692,7 @@ type Remotes struct {
 	fileWatch    chan string
 	cConns       uint16 // number of concurrent ssh connections, zero means unlimited
 	interactive  bool   // allow interactive prompting
-	sshPrivIdRsa string // path to ~/.ssh/id_rsa
+	sshPrivIDRsa string // path to ~/.ssh/id_rsa
 	caching      bool   // whether to try and cache the copy of the binary
 	depth        uint16 // depth of this node in the remote execution hierarchy
 	prefix       string // folder prefix to use for misc storage
@@ -702,6 +702,7 @@ type Remotes struct {
 	wg                 sync.WaitGroup       // keep track of each running SSH connection
 	lock               sync.Mutex           // mutex for access to sshmap
 	sshmap             map[string]*SSH      // map to each SSH struct with the remote as the key
+	running            chan struct{}        // closes when main loop is running
 	exiting            bool                 // flag to let us know if we're exiting
 	exitChan           chan struct{}        // closes when we should exit
 	semaphore          *semaphore.Semaphore // counting semaphore to limit concurrent connections
@@ -714,7 +715,7 @@ type Remotes struct {
 }
 
 // NewRemotes builds a Remotes struct.
-func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fileWatch chan string, cConns uint16, interactive bool, sshPrivIdRsa string, caching bool, depth uint16, prefix string, converger cv.Converger, convergerCb func(func(map[string]bool) error) (func(), error), flags Flags) *Remotes {
+func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fileWatch chan string, cConns uint16, interactive bool, sshPrivIDRsa string, caching bool, depth uint16, prefix string, converger cv.Converger, convergerCb func(func(map[string]bool) error) (func(), error), flags Flags) *Remotes {
 	return &Remotes{
 		clientURLs:   clientURLs,
 		remoteURLs:   remoteURLs,
@@ -723,13 +724,14 @@ func NewRemotes(clientURLs, remoteURLs []string, noop bool, remotes []string, fi
 		fileWatch:    fileWatch,
 		cConns:       cConns,
 		interactive:  interactive,
-		sshPrivIdRsa: sshPrivIdRsa,
+		sshPrivIDRsa: sshPrivIDRsa,
 		caching:      caching,
 		depth:        depth,
 		prefix:       prefix,
 		converger:    converger,
 		convergerCb:  convergerCb,
 		sshmap:       make(map[string]*SSH),
+		running:      make(chan struct{}),
 		exitChan:     make(chan struct{}),
 		semaphore:    semaphore.NewSemaphore(int(cConns)),
 		hostnames:    make([]string, len(remotes)),
@@ -830,18 +832,18 @@ func (obj *Remotes) NewSSH(file string) (*SSH, error) {
 
 // sshKeyAuth is a helper function to get the ssh key auth struct needed
 func (obj *Remotes) sshKeyAuth() (ssh.AuthMethod, error) {
-	if obj.sshPrivIdRsa == "" {
+	if obj.sshPrivIDRsa == "" {
 		return nil, fmt.Errorf("empty path specified")
 	}
 	p := ""
 	// TODO: this doesn't match strings of the form: ~james/.ssh/id_rsa
-	if strings.HasPrefix(obj.sshPrivIdRsa, "~/") {
+	if strings.HasPrefix(obj.sshPrivIDRsa, "~/") {
 		usr, err := user.Current()
 		if err != nil {
 			log.Printf("Remote: Can't find home directory automatically.")
 			return nil, err
 		}
-		p = path.Join(usr.HomeDir, obj.sshPrivIdRsa[len("~/"):])
+		p = path.Join(usr.HomeDir, obj.sshPrivIDRsa[len("~/"):])
 	}
 	if p == "" {
 		return nil, fmt.Errorf("empty path specified")
@@ -1022,11 +1024,17 @@ func (obj *Remotes) Run() {
 		}(sshobj, f)
 		obj.lock.Unlock()
 	}
+	close(obj.running) // notify
 }
 
+// Ready closes its returned channel when the Run method is up and ready. It is
+// useful to know when ready, since we often execute Run in a go routine.
+func (obj *Remotes) Ready() <-chan struct{} { return obj.running }
+
 // Exit causes as much of the Remotes struct to shutdown as quickly and as
 // cleanly as possible. It only returns once everything is shutdown.
 func (obj *Remotes) Exit() error {
+	<-obj.running // wait for Run to be finished before we exit!
 	obj.lock.Lock()
 	obj.exiting = true // don't spawn new ones once this flag is set!
 	obj.lock.Unlock()

resources/actions.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources

resources/augeas.go

@@ -1,25 +1,25 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 // +build !noaugeas
 
 package resources
 
 import (
-	"encoding/gob"
 	"fmt"
 	"log"
 	"os"
@@ -39,7 +39,6 @@ const (
 )
 
 func init() {
-	gob.Register(&AugeasRes{})
 	RegisterResource("augeas", func() Res { return &AugeasRes{} })
 }
 
@@ -94,7 +93,6 @@ func (obj *AugeasRes) Validate() error {
 
 // Init initiates the resource.
 func (obj *AugeasRes) Init() error {
-	obj.BaseRes.Kind = "augeas"
 	return obj.BaseRes.Init() // call base init, b/c we're overriding
 }
 

resources/augeas_disabled.go

@@ -1,19 +1,20 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 // +build noaugeas
 
 package resources

resources/autoedge.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources

resources/autogroup.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources

resources/autogroup_test.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources
@@ -108,6 +108,7 @@ func NewNoopResTest(name string) *NoopResTest {
 		NoopRes: NoopRes{
 			BaseRes: BaseRes{
 				Name: name,
+				Kind: "noop",
 				MetaParams: MetaParams{
 					AutoGroup: true, // always autogroup
 				},

resources/edge.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources

resources/exec.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources
@@ -20,10 +20,10 @@ package resources
 import (
 	"bufio"
 	"bytes"
-	"encoding/gob"
 	"fmt"
 	"log"
 	"os/exec"
+	"os/user"
 	"strings"
 	"sync"
 	"syscall"
@@ -34,7 +34,6 @@ import (
 )
 
 func init() {
-	gob.Register(&ExecRes{})
 	RegisterResource("exec", func() Res { return &ExecRes{} })
 }
 
@@ -48,6 +47,8 @@ type ExecRes struct {
 	WatchShell string  `yaml:"watchshell"` // the (optional) shell to use to run the watch cmd
 	IfCmd      string  `yaml:"ifcmd"`      // the if command to run
 	IfShell    string  `yaml:"ifshell"`    // the (optional) shell to use to run the if cmd
+	User       string  `yaml:"user"`       // the (optional) user to use to execute the command
+	Group      string  `yaml:"group"`      // the (optional) group to use to execute the command
 	Output     *string // all cmd output, read only, do not set!
 	Stdout     *string // the cmd stdout, read only, do not set!
 	Stderr     *string // the cmd stderr, read only, do not set!
@@ -68,12 +69,22 @@ func (obj *ExecRes) Validate() error {
 		return fmt.Errorf("command can't be empty")
 	}
 
+	// check that, if an user or a group is set, we're running as root
+	if obj.User != "" || obj.Group != "" {
+		currentUser, err := user.Current()
+		if err != nil {
+			return errwrap.Wrapf(err, "error looking up current user")
+		}
+		if currentUser.Uid != "0" {
+			return errwrap.Errorf("running as root is required if you want to use exec with a different user/group")
+		}
+	}
+
 	return obj.BaseRes.Validate()
 }
 
 // Init runs some startup code for this resource.
 func (obj *ExecRes) Init() error {
-	obj.BaseRes.Kind = "exec"
 	return obj.BaseRes.Init() // call base init, b/c we're overriding
 }
 
@@ -124,6 +135,12 @@ func (obj *ExecRes) Watch() error {
 			Pgid:    0,
 		}
 
+		// if we have a user and group, use them
+		var err error
+		if cmd.SysProcAttr.Credential, err = obj.getCredential(); err != nil {
+			return errwrap.Wrapf(err, "error while setting credential")
+		}
+
 		cmdReader, err := cmd.StdoutPipe()
 		if err != nil {
 			return errwrap.Wrapf(err, "error creating StdoutPipe for Cmd")
@@ -211,6 +228,13 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 			Setpgid: true,
 			Pgid:    0,
 		}
+
+		// if we have an user and group, use them
+		var err error
+		if cmd.SysProcAttr.Credential, err = obj.getCredential(); err != nil {
+			return false, errwrap.Wrapf(err, "error while setting credential")
+		}
+
 		if err := cmd.Run(); err != nil {
 			// TODO: check exit value
 			return true, nil // don't run
@@ -248,6 +272,12 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 		Pgid:    0,
 	}
 
+	// if we have a user and group, use them
+	var err error
+	if cmd.SysProcAttr.Credential, err = obj.getCredential(); err != nil {
+		return false, errwrap.Wrapf(err, "error while setting credential")
+	}
+
 	var out splitWriter
 	out.Init()
 	// from the docs: "If Stdout and Stderr are the same writer, at most one
@@ -266,7 +296,6 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
 	done := make(chan error)
 	go func() { done <- cmd.Wait() }()
 
-	var err error // error returned by cmd
 	select {
 	case e := <-done:
 		err = e // store
@@ -334,11 +363,39 @@ type ExecUID struct {
 	// TODO: add more elements here
 }
 
-// AutoEdges returns the AutoEdge interface. In this case no autoedges are used.
+// ExecResAutoEdges holds the state of the auto edge generator.
+type ExecResAutoEdges struct {
+	edges []ResUID
+}
+
+// Next returns the next automatic edge.
+func (obj *ExecResAutoEdges) Next() []ResUID {
+	return obj.edges
+}
+
+// Test gets results of the earlier Next() call, & returns if we should continue!
+func (obj *ExecResAutoEdges) Test(input []bool) bool {
+	return false // Never keep going
+	// TODO: We could return false if we find as many edges as the number of different path in cmdFiles()
+}
+
+// AutoEdges returns the AutoEdge interface. In this case the systemd units.
 func (obj *ExecRes) AutoEdges() (AutoEdge, error) {
-	// TODO: parse as many exec params to look for auto edges, for example
+	var data []ResUID
-	// the path of the binary in the Cmd variable might be from in a pkg
+	for _, x := range obj.cmdFiles() {
-	return nil, nil
+		var reversed = true
+		data = append(data, &PkgFileUID{
+			BaseUID: BaseUID{
+				Name:     obj.GetName(),
+				Kind:     obj.GetKind(),
+				Reversed: &reversed,
+			},
+			path: x, // what matters
+		})
+	}
+	return &ExecResAutoEdges{
+		edges: data,
+	}, nil
 }
 
 // UIDs includes all params to make a unique identification of this object.
@@ -397,6 +454,12 @@ func (obj *ExecRes) Compare(r Res) bool {
 	if obj.IfShell != res.IfShell {
 		return false
 	}
+	if obj.User != res.User {
+		return false
+	}
+	if obj.Group != res.Group {
+		return false
+	}
 
 	return true
 }
@@ -421,6 +484,37 @@ func (obj *ExecRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
+// getCredential returns the correct *syscall.Credential if an User and Group
+// are set.
+func (obj *ExecRes) getCredential() (*syscall.Credential, error) {
+	var uid, gid int
+	var err error
+	var currentUser *user.User
+	if currentUser, err = user.Current(); err != nil {
+		return nil, errwrap.Wrapf(err, "error looking up current user")
+	}
+	if currentUser.Uid != "0" {
+		// since we're not root, we've got nothing to do
+		return nil, nil
+	}
+
+	if obj.Group != "" {
+		gid, err = GetGID(obj.Group)
+		if err != nil {
+			return nil, errwrap.Wrapf(err, "error looking up gid for %s", obj.Group)
+		}
+	}
+
+	if obj.User != "" {
+		uid, err = GetUID(obj.User)
+		if err != nil {
+			return nil, errwrap.Wrapf(err, "error looking up uid for %s", obj.User)
+		}
+	}
+
+	return &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}, nil
+}
+
 // splitWriter mimics what the ssh.CombinedOutput command does, but stores the
 // the stdout and stderr separately. This is slightly tricky because we don't
 // want the combined output to be interleaved incorrectly. It creates sub writer
@@ -488,3 +582,24 @@ func (w *wrapWriter) Write(p []byte) (int, error) {
 func (w *wrapWriter) String() string {
 	return w.Buffer.String()
 }
+
+// cmdFiles returns all the potential files/commands this command might need.
+func (obj *ExecRes) cmdFiles() []string {
+	var paths []string
+	if obj.Shell != "" {
+		paths = append(paths, obj.Shell)
+	} else if cmdSplit := strings.Fields(obj.Cmd); len(cmdSplit) > 0 {
+		paths = append(paths, cmdSplit[0])
+	}
+	if obj.WatchShell != "" {
+		paths = append(paths, obj.WatchShell)
+	} else if watchSplit := strings.Fields(obj.WatchCmd); len(watchSplit) > 0 {
+		paths = append(paths, watchSplit[0])
+	}
+	if obj.IfShell != "" {
+		paths = append(paths, obj.IfShell)
+	} else if ifSplit := strings.Fields(obj.IfCmd); len(ifSplit) > 0 {
+		paths = append(paths, ifSplit[0])
+	}
+	return paths
+}

resources/exec_test.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources
@@ -25,13 +25,17 @@ func TestExecSendRecv1(t *testing.T) {
 	r1 := &ExecRes{
 		BaseRes: BaseRes{
 			Name:       "exec1",
-			//MetaParams: MetaParams,
+			Kind:       "exec",
+			MetaParams: DefaultMetaParams,
 		},
 		Cmd:   "echo hello world",
 		Shell: "/bin/bash",
 	}
 
 	r1.Setup(nil, r1, r1)
+	if err := r1.Validate(); err != nil {
+		t.Errorf("validate failed with: %v", err)
+	}
 	defer func() {
 		if err := r1.Close(); err != nil {
 			t.Errorf("close failed with: %v", err)
@@ -71,13 +75,17 @@ func TestExecSendRecv2(t *testing.T) {
 	r1 := &ExecRes{
 		BaseRes: BaseRes{
 			Name:       "exec1",
-			//MetaParams: MetaParams,
+			Kind:       "exec",
+			MetaParams: DefaultMetaParams,
 		},
 		Cmd:   "echo hello world 1>&2", // to stderr
 		Shell: "/bin/bash",
 	}
 
 	r1.Setup(nil, r1, r1)
+	if err := r1.Validate(); err != nil {
+		t.Errorf("validate failed with: %v", err)
+	}
 	defer func() {
 		if err := r1.Close(); err != nil {
 			t.Errorf("close failed with: %v", err)
@@ -117,13 +125,17 @@ func TestExecSendRecv3(t *testing.T) {
 	r1 := &ExecRes{
 		BaseRes: BaseRes{
 			Name:       "exec1",
-			//MetaParams: MetaParams,
+			Kind:       "exec",
+			MetaParams: DefaultMetaParams,
 		},
 		Cmd:   "echo hello world && echo goodbye world 1>&2", // to stdout && stderr
 		Shell: "/bin/bash",
 	}
 
 	r1.Setup(nil, r1, r1)
+	if err := r1.Validate(); err != nil {
+		t.Errorf("validate failed with: %v", err)
+	}
 	defer func() {
 		if err := r1.Close(); err != nil {
 			t.Errorf("close failed with: %v", err)

resources/file.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources
@@ -20,14 +20,12 @@ package resources
 import (
 	"bytes"
 	"crypto/sha256"
-	"encoding/gob"
 	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"os"
-	"os/user"
 	"path"
 	"path/filepath"
 	"strconv"
@@ -41,7 +39,6 @@ import (
 )
 
 func init() {
-	gob.Register(&FileRes{})
 	RegisterResource("file", func() Res { return &FileRes{} })
 }
 
@@ -99,11 +96,11 @@ func (obj *FileRes) Validate() error {
 		}
 	}
 
-	if _, err := obj.uid(); obj.Owner != "" && err != nil {
+	if _, err := GetUID(obj.Owner); obj.Owner != "" && err != nil {
 		return err
 	}
 
-	if _, err := obj.gid(); obj.Group != "" && err != nil {
+	if _, err := GetGID(obj.Group); obj.Group != "" && err != nil {
 		return err
 	}
 
@@ -126,29 +123,12 @@ func (obj *FileRes) mode() (os.FileMode, error) {
 	return os.FileMode(m), nil
 }
 
-// uid returns the user id for the owner specified in the yaml file graph.
-// Caller should first check obj.Owner is not empty
-func (obj *FileRes) uid() (int, error) {
-	u2, err2 := user.LookupId(obj.Owner)
-	if err2 == nil {
-		return strconv.Atoi(u2.Uid)
-	}
-
-	u, err := user.Lookup(obj.Owner)
-	if err == nil {
-		return strconv.Atoi(u.Uid)
-	}
-
-	return -1, errwrap.Wrapf(err, "owner lookup error (%s)", obj.Owner)
-}
-
 // Init runs some startup code for this resource.
 func (obj *FileRes) Init() error {
 	obj.sha256sum = ""
 	obj.path = obj.GetPath()                     // compute once
 	obj.isDir = strings.HasSuffix(obj.path, "/") // dirs have trailing slashes
 
-	obj.BaseRes.Kind = "file"
 	return obj.BaseRes.Init() // call base init, b/c we're overriding
 }
 
@@ -770,7 +750,7 @@ func (obj *FileRes) chownCheckApply(apply bool) (checkOK bool, _ error) {
 	}
 
 	if obj.Owner != "" {
-		expectedUID, err = obj.uid()
+		expectedUID, err = GetUID(obj.Owner)
 		if err != nil {
 			return false, err
 		}
@@ -780,7 +760,7 @@ func (obj *FileRes) chownCheckApply(apply bool) (checkOK bool, _ error) {
 	}
 
 	if obj.Group != "" {
-		expectedGID, err = obj.gid()
+		expectedGID, err = GetGID(obj.Group)
 		if err != nil {
 			return false, err
 		}

resources/file_attrs.go

@@ -1,43 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-// +build go1.7
-
-package resources
-
-import (
-	"os/user"
-	"strconv"
-
-	errwrap "github.com/pkg/errors"
-)
-
-// gid returns the group id for the group specified in the yaml file graph.
-// Caller should first check obj.Group is not empty
-func (obj *FileRes) gid() (int, error) {
-	g2, err2 := user.LookupGroupId(obj.Group)
-	if err2 == nil {
-		return strconv.Atoi(g2.Gid)
-	}
-
-	g, err := user.LookupGroup(obj.Group)
-	if err == nil {
-		return strconv.Atoi(g.Gid)
-	}
-
-	return -1, errwrap.Wrapf(err, "Group lookup error (%s)", obj.Group)
-}

resources/file_attrs_go1.6.go

@@ -1,43 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-// +build !go1.7
-
-package resources
-
-import (
-	"strconv"
-
-	group "github.com/hnakamur/group"
-	errwrap "github.com/pkg/errors"
-)
-
-// gid returns the group id for the group specified in the yaml file graph.
-// Caller should first check obj.Group is not empty
-func (obj *FileRes) gid() (int, error) {
-	g2, err2 := group.LookupId(obj.Group)
-	if err2 == nil {
-		return strconv.Atoi(g2.Gid)
-	}
-
-	g, err := group.Lookup(obj.Group)
-	if err == nil {
-		return strconv.Atoi(g.Gid)
-	}
-
-	return -1, errwrap.Wrapf(err, "Group lookup error (%s)", obj.Group)
-}

resources/file_test.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources

resources/graph.go

@@ -1,24 +1,23 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources
 
 import (
-	"encoding/gob"
 	"fmt"
 
 	"github.com/purpleidea/mgmt/pgraph"
@@ -29,7 +28,6 @@ import (
 
 func init() {
 	RegisterResource("graph", func() Res { return &GraphRes{} })
-	gob.Register(&GraphRes{})
 }
 
 // GraphRes is a resource that recursively runs a sub graph of resources.
@@ -89,7 +87,6 @@ func (obj *GraphRes) Init() error {
 		}
 	}
 
-	obj.BaseRes.Kind = "graph"
 	return obj.BaseRes.Init() // call base init, b/c we're overrriding
 }
 

resources/group.go

@@ -0,0 +1,314 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os/exec"
+	"os/user"
+	"strconv"
+	"syscall"
+
+	"github.com/purpleidea/mgmt/recwatch"
+
+	errwrap "github.com/pkg/errors"
+)
+
+func init() {
+	RegisterResource("group", func() Res { return &GroupRes{} })
+}
+
+const groupFile = "/etc/group"
+
+// GroupRes is a user group resource.
+type GroupRes struct {
+	BaseRes `yaml:",inline"`
+	State   string  `yaml:"state"` // state: exists, absent
+	GID     *uint32 `yaml:"gid"`   // the group's gid
+
+	recWatcher *recwatch.RecWatcher
+}
+
+// Default returns some sensible defaults for this resource.
+func (obj *GroupRes) Default() Res {
+	return &GroupRes{
+		BaseRes: BaseRes{
+			MetaParams: DefaultMetaParams, // force a default
+		},
+	}
+}
+
+// Validate if the params passed in are valid data.
+func (obj *GroupRes) Validate() error {
+	if obj.State != "exists" && obj.State != "absent" {
+		return fmt.Errorf("State must be 'exists' or 'absent'")
+	}
+	return obj.BaseRes.Validate()
+}
+
+// Init initializes the resource.
+func (obj *GroupRes) Init() error {
+	return obj.BaseRes.Init() // call base init, b/c we're overriding
+}
+
+// Watch is the primary listener for this resource and it outputs events.
+func (obj *GroupRes) Watch() error {
+	var err error
+	obj.recWatcher, err = recwatch.NewRecWatcher(groupFile, false)
+	if err != nil {
+		return err
+	}
+	defer obj.recWatcher.Close()
+
+	// notify engine that we're running
+	if err := obj.Running(); err != nil {
+		return err // bubble up a NACK...
+	}
+
+	var send = false // send event?
+	var exit *error
+
+	for {
+		if obj.debug {
+			log.Printf("%s: Watching: %s", obj, groupFile) // attempting to watch...
+		}
+
+		select {
+		case event, ok := <-obj.recWatcher.Events():
+			if !ok { // channel shutdown
+				return nil
+			}
+			if err := event.Error; err != nil {
+				return errwrap.Wrapf(err, "Unknown %s watcher error", obj)
+			}
+			if obj.debug { // don't access event.Body if event.Error isn't nil
+				log.Printf("%s: Event(%s): %v", obj, event.Body.Name, event.Body.Op)
+			}
+			send = true
+			obj.StateOK(false) // dirty
+
+		case event := <-obj.Events():
+			if exit, send = obj.ReadEvent(event); exit != nil {
+				return *exit // exit
+			}
+			//obj.StateOK(false) // dirty // these events don't invalidate state
+		}
+
+		// do all our event sending all together to avoid duplicate msgs
+		if send {
+			send = false
+			obj.Event()
+		}
+	}
+}
+
+// CheckApply method for Group resource.
+func (obj *GroupRes) CheckApply(apply bool) (checkOK bool, err error) {
+	log.Printf("%s: CheckApply(%t)", obj, apply)
+
+	// check if the group exists
+	exists := true
+	group, err := user.LookupGroup(obj.GetName())
+	if err != nil {
+		if _, ok := err.(user.UnknownGroupError); !ok {
+			return false, errwrap.Wrapf(err, "error looking up group")
+		}
+		exists = false
+	}
+	// if the group doesn't exist and should be absent, we are done
+	if obj.State == "absent" && !exists {
+		return true, nil
+	}
+	// if the group exists and no GID is specified, we are done
+	if obj.State == "exists" && exists && obj.GID == nil {
+		return true, nil
+	}
+	if exists && obj.GID != nil {
+		// check if GID is taken
+		lookupGID, err := user.LookupGroupId(strconv.Itoa(int(*obj.GID)))
+		if err != nil {
+			if _, ok := err.(user.UnknownGroupIdError); !ok {
+				return false, errwrap.Wrapf(err, "error looking up GID")
+			}
+		}
+		if lookupGID != nil && lookupGID.Name != obj.GetName() {
+			return false, fmt.Errorf("the requested GID belongs to another group")
+		}
+		// get the existing group's GID
+		existingGID, err := strconv.ParseUint(group.Gid, 10, 32)
+		if err != nil {
+			return false, errwrap.Wrapf(err, "error casting existing GID")
+		}
+		// check if existing group has the wrong GID
+		// if it is wrong groupmod will change it to the desired value
+		if *obj.GID != uint32(existingGID) {
+			log.Printf("%s: Inconsistent GID: %s", obj, obj.GetName())
+		}
+		// if the group exists and has the correct GID, we are done
+		if obj.State == "exists" && *obj.GID == uint32(existingGID) {
+			return true, nil
+		}
+	}
+
+	if !apply {
+		return false, nil
+	}
+
+	var cmdName string
+	args := []string{obj.GetName()}
+
+	if obj.State == "exists" {
+		if exists {
+			log.Printf("%s: Modifying group: %s", obj, obj.GetName())
+			cmdName = "groupmod"
+		} else {
+			log.Printf("%s: Adding group: %s", obj, obj.GetName())
+			cmdName = "groupadd"
+		}
+		if obj.GID != nil {
+			args = append(args, "-g", fmt.Sprintf("%d", *obj.GID))
+		}
+	}
+	if obj.State == "absent" && exists {
+		log.Printf("%s: Deleting group: %s", obj, obj.GetName())
+		cmdName = "groupdel"
+	}
+
+	cmd := exec.Command(cmdName, args...)
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+		Pgid:    0,
+	}
+
+	// open a pipe to get error messages from os/exec
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return false, errwrap.Wrapf(err, "failed to initialize stderr pipe")
+	}
+
+	// start the command
+	if err := cmd.Start(); err != nil {
+		return false, errwrap.Wrapf(err, "cmd failed to start")
+	}
+	// capture any error messages
+	slurp, err := ioutil.ReadAll(stderr)
+	if err != nil {
+		return false, errwrap.Wrapf(err, "error slurping error message")
+	}
+	// wait until cmd exits and return error message if any
+	if err := cmd.Wait(); err != nil {
+		return false, errwrap.Wrapf(err, "%s", slurp)
+	}
+
+	return false, nil
+}
+
+// GroupUID is the UID struct for GroupRes.
+type GroupUID struct {
+	BaseUID
+	name string
+	gid  *uint32
+}
+
+// IFF aka if and only if they are equivalent, return true. If not, false.
+func (obj *GroupUID) IFF(uid ResUID) bool {
+	res, ok := uid.(*GroupUID)
+	if !ok {
+		return false
+	}
+	if obj.gid != nil && res.gid != nil {
+		if *obj.gid != *res.gid {
+			return false
+		}
+	}
+	if obj.name != "" && res.name != "" {
+		if obj.name != res.name {
+			return false
+		}
+	}
+	return true
+}
+
+// UIDs includes all params to make a unique identification of this object.
+// Most resources only return one, although some resources can return multiple.
+func (obj *GroupRes) UIDs() []ResUID {
+	x := &GroupUID{
+		BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
+		name:    obj.Name,
+		gid:     obj.GID,
+	}
+	return []ResUID{x}
+}
+
+// GroupCmp returns whether two resources can be grouped together or not.
+func (obj *GroupRes) GroupCmp(r Res) bool {
+	_, ok := r.(*GroupRes)
+	if !ok {
+		return false
+	}
+	return false
+}
+
+// Compare two resources and return if they are equivalent.
+func (obj *GroupRes) Compare(r Res) bool {
+	// we can only compare GroupRes to others of the same resource kind
+	res, ok := r.(*GroupRes)
+	if !ok {
+		return false
+	}
+	if !obj.BaseRes.Compare(res) { // call base Compare
+		return false
+	}
+	if obj.Name != res.Name {
+		return false
+	}
+	if obj.State != res.State {
+		return false
+	}
+	if (obj.GID == nil) != (res.GID == nil) {
+		return false
+	}
+	if obj.GID != nil && res.GID != nil {
+		if *obj.GID != *res.GID {
+			return false
+		}
+	}
+	return true
+}
+
+// UnmarshalYAML is the custom unmarshal handler for this struct.
+// It is primarily useful for setting the defaults.
+func (obj *GroupRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type rawRes GroupRes // indirection to avoid infinite recursion
+
+	def := obj.Default()       // get the default
+	res, ok := def.(*GroupRes) // put in the right format
+	if !ok {
+		return fmt.Errorf("could not convert to GroupRes")
+	}
+	raw := rawRes(*res) // convert; the defaults go here
+
+	if err := unmarshal(&raw); err != nil {
+		return err
+	}
+
+	*obj = GroupRes(raw) // restore from indirection with type conversion!
+	return nil
+}

resources/hostname.go

@@ -1,24 +1,23 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources
 
 import (
-	"encoding/gob"
 	"errors"
 	"fmt"
 	"log"
@@ -36,7 +35,6 @@ var ErrResourceInsufficientParameters = errors.New(
 
 func init() {
 	RegisterResource("hostname", func() Res { return &HostnameRes{} })
-	gob.Register(&HostnameRes{})
 }
 
 const (
@@ -88,7 +86,6 @@ func (obj *HostnameRes) Validate() error {
 
 // Init runs some startup code for this resource.
 func (obj *HostnameRes) Init() error {
-	obj.BaseRes.Kind = "hostname"
 	if obj.PrettyHostname == "" {
 		obj.PrettyHostname = obj.Hostname
 	}

resources/kv.go

@@ -1,24 +1,23 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources
 
 import (
-	"encoding/gob"
 	"fmt"
 	"log"
 	"strconv"
@@ -28,7 +27,6 @@ import (
 
 func init() {
 	RegisterResource("kv", func() Res { return &KVRes{} })
-	gob.Register(&KVRes{})
 }
 
 // KVResSkipCmpStyle represents the different styles of comparison when using SkipLessThan.
@@ -89,7 +87,6 @@ func (obj *KVRes) Validate() error {
 
 // Init initializes the resource.
 func (obj *KVRes) Init() error {
-	obj.BaseRes.Kind = "kv"
 	return obj.BaseRes.Init() // call base init, b/c we're overriding
 }
 

resources/metaparams.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources

resources/mgraph.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources // TODO: can this be a separate package or will it break the dag?

resources/msg.go

@@ -1,24 +1,23 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources
 
 import (
-	"encoding/gob"
 	"fmt"
 	"log"
 	"regexp"
@@ -29,7 +28,6 @@ import (
 
 func init() {
 	RegisterResource("msg", func() Res { return &MsgRes{} })
-	gob.Register(&MsgRes{})
 }
 
 // MsgRes is a resource that writes messages to logs.
@@ -71,12 +69,23 @@ func (obj *MsgRes) Validate() error {
 			return fmt.Errorf("fields cannot begin with _")
 		}
 	}
+	switch obj.Priority {
+	case "Emerg":
+	case "Alert":
+	case "Crit":
+	case "Err":
+	case "Warning":
+	case "Notice":
+	case "Info":
+	case "Debug":
+	default:
+		return fmt.Errorf("invalid Priority '%s'", obj.Priority)
+	}
 	return obj.BaseRes.Validate()
 }
 
 // Init runs some startup code for this resource.
 func (obj *MsgRes) Init() error {
-	obj.BaseRes.Kind = "msg"
 	return obj.BaseRes.Init() // call base init, b/c we're overrriding
 }
 
@@ -97,7 +106,6 @@ func (obj *MsgRes) updateStateOK() {
 }
 
 // JournalPriority converts a string description to a numeric priority.
-// XXX: Have Validate() make sure it actually is one of these.
 func (obj *MsgRes) journalPriority() journal.Priority {
 	switch obj.Priority {
 	case "Emerg":

resources/msg_test.go

@@ -0,0 +1,54 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package resources
+
+import (
+	"testing"
+)
+
+func TestMsgValidate1(t *testing.T) {
+	r1 := &MsgRes{
+		BaseRes: BaseRes{
+			Name:       "msg1",
+			Kind:       "msg",
+			MetaParams: DefaultMetaParams,
+		},
+		Priority: "Debug",
+	}
+
+	r1.Setup(nil, r1, r1)
+	if err := r1.Validate(); err != nil {
+		t.Errorf("validate failed with: %v", err)
+	}
+}
+
+func TestMsgValidate2(t *testing.T) {
+	r1 := &MsgRes{
+		BaseRes: BaseRes{
+			Name:       "msg1",
+			Kind:       "msg",
+			MetaParams: DefaultMetaParams,
+		},
+		Priority: "UnrealPriority",
+	}
+
+	r1.Setup(nil, r1, r1)
+	if err := r1.Validate(); err == nil {
+		t.Errorf("validation error is nil")
+	}
+}

resources/noop.go

@@ -1,31 +1,29 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources
 
 import (
-	"encoding/gob"
 	"fmt"
 	"log"
 )
 
 func init() {
 	RegisterResource("noop", func() Res { return &NoopRes{} })
-	gob.Register(&NoopRes{})
 }
 
 // NoopRes is a no-op resource that does nothing.
@@ -50,7 +48,6 @@ func (obj *NoopRes) Validate() error {
 
 // Init runs some startup code for this resource.
 func (obj *NoopRes) Init() error {
-	obj.BaseRes.Kind = "noop"
 	return obj.BaseRes.Init() // call base init, b/c we're overriding
 }
 

resources/nspawn.go

@@ -1,34 +1,36 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package resources
 
 import (
-	"encoding/gob"
 	"errors"
 	"fmt"
 	"log"
+	"strconv"
+	"unicode"
 
 	"github.com/purpleidea/mgmt/util"
 
+	systemdDbus "github.com/coreos/go-systemd/dbus"
+	machined "github.com/coreos/go-systemd/machine1"
 	systemdUtil "github.com/coreos/go-systemd/util"
 	"github.com/godbus/dbus"
 	errwrap "github.com/pkg/errors"
-	machined "github.com/purpleidea/go-systemd/machine1"
 )
 
 const (
@@ -42,15 +44,14 @@ const (
 
 func init() {
 	RegisterResource("nspawn", func() Res { return &NspawnRes{} })
-	gob.Register(&NspawnRes{})
 }
 
 // NspawnRes is an nspawn container resource.
 type NspawnRes struct {
 	BaseRes `yaml:",inline"`
 	State   string `yaml:"state"`
-	// We're using the svc resource to start the machine because that's
+	// We're using the svc resource to start and stop the machine because
-	// what machinectl does. We're not using svc.Watch because then we
+	// that's what machinectl does. We're not using svc.Watch because then we
 	// would have two watches potentially racing each other and producing
 	// potentially unexpected results. We get everything we need to monitor
 	// the machine state changes from the org.freedesktop.machine1 object.
@@ -67,33 +68,90 @@ func (obj *NspawnRes) Default() Res {
 	}
 }
 
+// makeComposite creates a pointer to a SvcRes. The pointer is used to
+// validate and initialize the nested svc.
+func (obj *NspawnRes) makeComposite() (*SvcRes, error) {
+	res, err := NewNamedResource("svc", fmt.Sprintf(nspawnServiceTmpl, obj.GetName()))
+	if err != nil {
+		return nil, err
+	}
+	svc := res.(*SvcRes)
+	svc.State = obj.State
+	return svc, nil
+}
+
+// systemdVersion uses dbus to check which version of systemd is installed.
+func systemdVersion() (uint16, error) {
+	// check if systemd is running
+	if !systemdUtil.IsRunningSystemd() {
+		return 0, fmt.Errorf("systemd is not running")
+	}
+	bus, err := systemdDbus.NewSystemdConnection()
+	if err != nil {
+		return 0, errwrap.Wrapf(err, "failed to connect to bus")
+	}
+	defer bus.Close()
+	// get the systemd version
+	verString, err := bus.GetManagerProperty("Version")
+	if err != nil {
+		return 0, errwrap.Wrapf(err, "could not get version property")
+	}
+	// lose the surrounding quotes
+	verNum, err := strconv.Unquote(verString)
+	if err != nil {
+		return 0, errwrap.Wrapf(err, "error unquoting version number")
+	}
+	// cast to uint16
+	ver, err := strconv.ParseUint(verNum, 10, 16)
+	if err != nil {
+		return 0, errwrap.Wrapf(err, "error casting systemd version number")
+	}
+	return uint16(ver), nil
+}
+
 // Validate if the params passed in are valid data.
 func (obj *NspawnRes) Validate() error {
-	// TODO: validStates should be an enum!
+	if len(obj.GetName()) > 64 {
-	validStates := map[string]struct{}{
+		return fmt.Errorf("name must be 64 characters or less")
-		stopped: {},
-		running: {},
 	}
-	if _, exists := validStates[obj.State]; !exists {
+	// check if systemd version is higher than 231 to allow non-alphanumeric
+	// machine names, as previous versions would error in such cases
+	ver, err := systemdVersion()
+	if err != nil {
+		return err
+	}
+	if ver < 231 {
+		for _, char := range obj.GetName() {
+			if !unicode.IsLetter(char) && !unicode.IsNumber(char) {
+				return fmt.Errorf("name must only contain alphanumeric characters for systemd versions < 231")
+			}
+		}
+	}
+
+	if obj.State != running && obj.State != stopped {
 		return fmt.Errorf("invalid state: %s", obj.State)
 	}
 
-	if err := obj.svc.Validate(); err != nil { // composite resource
+	svc, err := obj.makeComposite()
-		return errwrap.Wrapf(err, "validate failed for embedded svc")
+	if err != nil {
+		return errwrap.Wrapf(err, "makeComposite failed in validate")
+	}
+	if err := svc.Validate(); err != nil { // composite resource
+		return errwrap.Wrapf(err, "validate failed for embedded svc: %s", obj.svc)
 	}
 	return obj.BaseRes.Validate()
 }
 
 // Init runs some startup code for this resource.
 func (obj *NspawnRes) Init() error {
-	var serviceName = fmt.Sprintf(nspawnServiceTmpl, obj.GetName())
+	svc, err := obj.makeComposite()
-	obj.svc = &SvcRes{}
+	if err != nil {
-	obj.svc.Name = serviceName
+		return errwrap.Wrapf(err, "makeComposite failed in init")
-	obj.svc.State = obj.State
+	}
+	obj.svc = svc
 	if err := obj.svc.Init(); err != nil {
 		return err
 	}
-	obj.BaseRes.Kind = "nspawn"
 	return obj.BaseRes.Init()
 }
 
@@ -118,8 +176,10 @@ func (obj *NspawnRes) Watch() error {
 	if err := call.Err; err != nil {
 		return err
 	}
-	buschan := make(chan *dbus.Signal, 10)
+	// TODO: verify that implementation doesn't deadlock if there are unread
-	bus.Signal(buschan)
+	// messages left in the channel
+	busChan := make(chan *dbus.Signal, 10)
+	bus.Signal(busChan)
 
 	// notify engine that we're running
 	if err := obj.Running(); err != nil {
@@ -129,9 +189,12 @@ func (obj *NspawnRes) Watch() error {
 	var send = false
 	var exit *error
 
+	defer close(busChan)
+	defer bus.Close()
+	defer bus.RemoveSignal(busChan)
 	for {
 		select {
-		case event := <-buschan:
+		case event := <-busChan:
 			// process org.freedesktop.machine1 events for this resource's name
 			if event.Body[0] == obj.GetName() {
 				log.Printf("%s: Event received: %v", obj, event.Name)
@@ -178,7 +241,7 @@ func (obj *NspawnRes) CheckApply(apply bool) (checkOK bool, err error) {
 	// compare the current state with the desired state and perform the
 	// appropriate action
 	var exists = true
-	properties, err := conn.GetProperties(obj.GetName())
+	properties, err := conn.DescribeMachine(obj.GetName())
 	if err != nil {
 		if err, ok := err.(dbus.Error); ok && err.Name !=
 			"org.freedesktop.machine1.NoSuchMachine" {
@@ -211,26 +274,11 @@ func (obj *NspawnRes) CheckApply(apply bool) (checkOK bool, err error) {
 		return false, nil
 	}
 
-	if obj.debug {
 	log.Printf("%s: CheckApply() applying '%s' state", obj, obj.State)
-	}
+	// use the embedded svc to apply the correct state
-
-	if obj.State == running {
-		// start the machine using svc resource
-		log.Printf("%s: Starting machine", obj)
-		// assume state had to be changed at this point, ignore checkOK
 	if _, err := obj.svc.CheckApply(apply); err != nil {
 		return false, errwrap.Wrapf(err, "nested svc failed")
 	}
-	}
-	if obj.State == stopped {
-		// terminate the machine with
-		// org.freedesktop.machine1.Manager.KillMachine
-		log.Printf("%s: Stopping machine", obj)
-		if err := conn.TerminateMachine(obj.GetName()); err != nil {
-			return false, errwrap.Wrapf(err, "failed to stop machine")
-		}
-	}
 
 	return false, nil
 }
@@ -287,6 +335,9 @@ func (obj *NspawnRes) Compare(r Res) bool {
 	if obj.Name != res.Name {
 		return false
 	}
+	if obj.State != res.State {
+		return false
+	}
 
 	if !obj.svc.Compare(res.svc) {
 		return false

resources/packagekit/packagekit.go

@@ -1,18 +1,18 @@
 // Mgmt
-// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
 // Written by James Shubin <james@shubin.ca> and the project contributors
 //
 // This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
+// it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
+// GNU General Public License for more details.
 //
-// You should have received a copy of the GNU Affero General Public License
+// You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 // Package packagekit provides an interface to interact with packagekit.
@@ -33,8 +33,8 @@ import (
 
 // global tweaks of verbosity and code path
 const (
-	PK_DEBUG = false
+	Debug    = false
-	PARANOID = false // enable if you see any ghosts
+	Paranoid = false // enable if you see any ghosts
 )
 
 // constants which might need to be tweaked or which contain special dbus strings.
@@ -74,76 +74,76 @@ var (
 //type enum_filter uint64
 // https://github.com/hughsie/PackageKit/blob/master/lib/packagekit-glib2/pk-enum.c
 const ( //static const PkEnumMatch enum_filter[]
-	PK_FILTER_ENUM_UNKNOWN         uint64 = 1 << iota // "unknown"
+	PkFilterEnumUnknown        uint64 = 1 << iota // "unknown"
-	PK_FILTER_ENUM_NONE                               // "none"
+	PkFilterEnumNone                              // "none"
-	PK_FILTER_ENUM_INSTALLED                          // "installed"
+	PkFilterEnumInstalled                         // "installed"
-	PK_FILTER_ENUM_NOT_INSTALLED                      // "~installed"
+	PkFilterEnumNotInstalled                      // "~installed"
-	PK_FILTER_ENUM_DEVELOPMENT                        // "devel"
+	PkFilterEnumDevelopment                       // "devel"
-	PK_FILTER_ENUM_NOT_DEVELOPMENT                    // "~devel"
+	PkFilterEnumNotDevelopment                    // "~devel"
-	PK_FILTER_ENUM_GUI                                // "gui"
+	PkFilterEnumGui                               // "gui"
-	PK_FILTER_ENUM_NOT_GUI                            // "~gui"
+	PkFilterEnumNotGui                            // "~gui"
-	PK_FILTER_ENUM_FREE                               // "free"
+	PkFilterEnumFree                              // "free"
-	PK_FILTER_ENUM_NOT_FREE                           // "~free"
+	PkFilterEnumNotFree                           // "~free"
-	PK_FILTER_ENUM_VISIBLE                            // "visible"
+	PkFilterEnumVisible                           // "visible"
-	PK_FILTER_ENUM_NOT_VISIBLE                        // "~visible"
+	PkFilterEnumNotVisible                        // "~visible"
-	PK_FILTER_ENUM_SUPPORTED                          // "supported"
+	PkFilterEnumSupported                         // "supported"
-	PK_FILTER_ENUM_NOT_SUPPORTED                      // "~supported"
+	PkFilterEnumNotSupported                      // "~supported"
-	PK_FILTER_ENUM_BASENAME                           // "basename"
+	PkFilterEnumBasename                          // "basename"
-	PK_FILTER_ENUM_NOT_BASENAME                       // "~basename"
+	PkFilterEnumNotBasename                       // "~basename"
-	PK_FILTER_ENUM_NEWEST                             // "newest"
+	PkFilterEnumNewest                            // "newest"
-	PK_FILTER_ENUM_NOT_NEWEST                         // "~newest"
+	PkFilterEnumNotNewest                         // "~newest"
-	PK_FILTER_ENUM_ARCH                               // "arch"
+	PkFilterEnumArch                              // "arch"
-	PK_FILTER_ENUM_NOT_ARCH                           // "~arch"
+	PkFilterEnumNotArch                           // "~arch"
-	PK_FILTER_ENUM_SOURCE                             // "source"
+	PkFilterEnumSource                            // "source"
-	PK_FILTER_ENUM_NOT_SOURCE                         // "~source"
+	PkFilterEnumNotSource                         // "~source"
-	PK_FILTER_ENUM_COLLECTIONS                        // "collections"
+	PkFilterEnumCollections                       // "collections"
-	PK_FILTER_ENUM_NOT_COLLECTIONS                    // "~collections"
+	PkFilterEnumNotCollections                    // "~collections"
-	PK_FILTER_ENUM_APPLICATION                        // "application"
+	PkFilterEnumApplication                       // "application"
-	PK_FILTER_ENUM_NOT_APPLICATION                    // "~application"
+	PkFilterEnumNotApplication                    // "~application"
-	PK_FILTER_ENUM_DOWNLOADED                         // "downloaded"
+	PkFilterEnumDownloaded                        // "downloaded"
-	PK_FILTER_ENUM_NOT_DOWNLOADED                     // "~downloaded"
+	PkFilterEnumNotDownloaded                     // "~downloaded"
 )
 
 // constants from packagekit c library.
 const ( //static const PkEnumMatch enum_transaction_flag[]
-	PK_TRANSACTION_FLAG_ENUM_NONE            uint64 = 1 << iota // "none"
+	PkTransactionFlagEnumNone           uint64 = 1 << iota // "none"
-	PK_TRANSACTION_FLAG_ENUM_ONLY_TRUSTED                       // "only-trusted"
+	PkTransactionFlagEnumOnlyTrusted                       // "only-trusted"
-	PK_TRANSACTION_FLAG_ENUM_SIMULATE                           // "simulate"
+	PkTransactionFlagEnumSimulate                          // "simulate"
-	PK_TRANSACTION_FLAG_ENUM_ONLY_DOWNLOAD                      // "only-download"
+	PkTransactionFlagEnumOnlyDownload                      // "only-download"
-	PK_TRANSACTION_FLAG_ENUM_ALLOW_REINSTALL                    // "allow-reinstall"
+	PkTransactionFlagEnumAllowReinstall                    // "allow-reinstall"
-	PK_TRANSACTION_FLAG_ENUM_JUST_REINSTALL                     // "just-reinstall"
+	PkTransactionFlagEnumJustReinstall                     // "just-reinstall"
-	PK_TRANSACTION_FLAG_ENUM_ALLOW_DOWNGRADE                    // "allow-downgrade"
+	PkTransactionFlagEnumAllowDowngrade                    // "allow-downgrade"
 )
 
 // constants from packagekit c library.
 const ( //typedef enum
-	PK_INFO_ENUM_UNKNOWN uint64 = 1 << iota
+	PkInfoEnumUnknown uint64 = 1 << iota
-	PK_INFO_ENUM_INSTALLED
+	PkInfoEnumInstalled
-	PK_INFO_ENUM_AVAILABLE
+	PkInfoEnumAvailable
-	PK_INFO_ENUM_LOW
+	PkInfoEnumLow
-	PK_INFO_ENUM_ENHANCEMENT
+	PkInfoEnumEnhancement
-	PK_INFO_ENUM_NORMAL
+	PkInfoEnumNormal
-	PK_INFO_ENUM_BUGFIX
+	PkInfoEnumBugfix
-	PK_INFO_ENUM_IMPORTANT
+	PkInfoEnumImportant
-	PK_INFO_ENUM_SECURITY
+	PkInfoEnumSecurity
-	PK_INFO_ENUM_BLOCKED
+	PkInfoEnumBlocked
-	PK_INFO_ENUM_DOWNLOADING
+	PkInfoEnumDownloading
-	PK_INFO_ENUM_UPDATING
+	PkInfoEnumUpdating
-	PK_INFO_ENUM_INSTALLING
+	PkInfoEnumInstalling
-	PK_INFO_ENUM_REMOVING
+	PkInfoEnumRemoving
-	PK_INFO_ENUM_CLEANUP
+	PkInfoEnumCleanup
-	PK_INFO_ENUM_OBSOLETING
+	PkInfoEnumObsoleting
-	PK_INFO_ENUM_COLLECTION_INSTALLED
+	PkInfoEnumCollectionInstalled
-	PK_INFO_ENUM_COLLECTION_AVAILABLE
+	PkInfoEnumCollectionAvailable
-	PK_INFO_ENUM_FINISHED
+	PkInfoEnumFinished
-	PK_INFO_ENUM_REINSTALLING
+	PkInfoEnumReinstalling
-	PK_INFO_ENUM_DOWNGRADING
+	PkInfoEnumDowngrading
-	PK_INFO_ENUM_PREPARING
+	PkInfoEnumPreparing
-	PK_INFO_ENUM_DECOMPRESSING
+	PkInfoEnumDecompressing
-	PK_INFO_ENUM_UNTRUSTED
+	PkInfoEnumUntrusted
-	PK_INFO_ENUM_TRUSTED
+	PkInfoEnumTrusted
-	PK_INFO_ENUM_UNAVAILABLE
+	PkInfoEnumUnavailable
-	PK_INFO_ENUM_LAST
+	PkInfoEnumLast
 )
 
 // Conn is a wrapper struct so we can pass bus connection around in the struct.
@@ -184,7 +184,7 @@ func (bus *Conn) Close() error {
 
 // internal helper to add signal matches to the bus, should only be called once
 func (bus *Conn) matchSignal(ch chan *dbus.Signal, path dbus.ObjectPath, iface string, signals []string) error {
-	if PK_DEBUG {
+	if Debug {
 		log.Printf("PackageKit: matchSignal(%v, %v, %v, %v)", ch, path, iface, signals)
 	}
 	// eg: gdbus monitor --system --dest org.freedesktop.PackageKit --object-path /org/freedesktop/PackageKit | grep <signal>
@@ -224,7 +224,7 @@ func (bus *Conn) WatchChanges() (chan *dbus.Signal, error) {
 	if err != nil {
 		return nil, err
 	}
-	if PARANOID { // TODO: this filtering might not be necessary anymore...
+	if Paranoid { // TODO: this filtering might not be necessary anymore...
 		// try to handle the filtering inside this function!
 		rch := make(chan *dbus.Signal)
 		go func() {
@@ -257,7 +257,7 @@ func (bus *Conn) WatchChanges() (chan *dbus.Signal, error) {
 
 // CreateTransaction creates and returns a transaction path.
 func (bus *Conn) CreateTransaction() (dbus.ObjectPath, error) {
-	if PK_DEBUG {
+	if Debug {
 		log.Println("PackageKit: CreateTransaction()")
 	}
 	var interfacePath dbus.ObjectPath
@@ -266,7 +266,7 @@ func (bus *Conn) CreateTransaction() (dbus.ObjectPath, error) {
 	if call != nil {
 		return "", call
 	}
-	if PK_DEBUG {
+	if Debug {
 		log.Printf("PackageKit: CreateTransaction(): %v", interfacePath)
 	}
 	return interfacePath, nil
@@ -284,12 +284,12 @@ func (bus *Conn) ResolvePackages(packages []string, filter uint64) ([]string, er
 	// add signal matches for Package and Finished which will always be last
 	var signals = []string{"Package", "Finished", "Error", "Destroy"}
 	bus.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
-	if PK_DEBUG {
+	if Debug {
 		log.Printf("PackageKit: ResolvePackages(): Object(%v, %v)", PkIface, interfacePath)
 	}
 	obj := bus.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
 	call := obj.Call(FmtTransactionMethod("Resolve"), 0, filter, packages)
-	if PK_DEBUG {
+	if Debug {
 		log.Println("PackageKit: ResolvePackages(): Call: Success!")
 	}
 	if call.Err != nil {
@@ -300,7 +300,7 @@ loop:
 		// FIXME: add a timeout option to error in case signals are dropped!
 		select {
 		case signal := <-ch:
-			if PK_DEBUG {
+			if Debug {
 				log.Printf("PackageKit: ResolvePackages(): Signal: %+v", signal)
 			}
 			if signal.Path != interfacePath {
@@ -339,7 +339,7 @@ loop:
 // IsInstalledList queries a list of packages to see if they are installed.
 func (bus *Conn) IsInstalledList(packages []string) ([]bool, error) {
 	var filter uint64          // initializes at the "zero" value of 0
-	filter += PK_FILTER_ENUM_ARCH // always search in our arch
+	filter += PkFilterEnumArch // always search in our arch
 	packageIDs, e := bus.ResolvePackages(packages, filter)
 	if e != nil {
 		return nil, fmt.Errorf("ResolvePackages error: %v", e)
@@ -396,7 +396,11 @@ func (bus *Conn) InstallPackages(packageIDs []string, transactionFlags uint64) e
 	bus.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
 
 	obj := bus.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
-	call := obj.Call(FmtTransactionMethod("InstallPackages"), 0, transactionFlags, packageIDs)
+	call := obj.Call(FmtTransactionMethod("RefreshCache"), 0, false)
+	if call.Err != nil {
+		return call.Err
+	}
+	call = obj.Call(FmtTransactionMethod("InstallPackages"), 0, transactionFlags, packageIDs)
 	if call.Err != nil {
 		return call.Err
 	}
@@ -593,7 +597,7 @@ loop:
 
 // GetUpdates gets a list of packages that are installed and which can be updated, mod filter.
 func (bus *Conn) GetUpdates(filter uint64) ([]string, error) {
-	if PK_DEBUG {
+	if Debug {
 		log.Println("PackageKit: GetUpdates()")
 	}
 	packageIDs := []string{}
@@ -664,11 +668,11 @@ func (bus *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 		count++
 	}
 
-	if !(filter&PK_FILTER_ENUM_ARCH == PK_FILTER_ENUM_ARCH) {
+	if !(filter&PkFilterEnumArch == PkFilterEnumArch) {
-		filter += PK_FILTER_ENUM_ARCH // always search in our arch
+		filter += PkFilterEnumArch // always search in our arch
 	}
 
-	if PK_DEBUG {
+	if Debug {
 		log.Printf("PackageKit: PackagesToPackageIDs(): %v", strings.Join(packages, ", "))
 	}
 	resolved, e := bus.ResolvePackages(packages, filter)
@@ -771,7 +775,7 @@ func (bus *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 	// this check is for packages that need to verify their "newest" status
 	// we need to know this so we can install the correct newest packageID!
 	recursion := make(map[string]*PkPackageIDActionData)
-	if !(filter&PK_FILTER_ENUM_NEWEST == PK_FILTER_ENUM_NEWEST) {
+	if !(filter&PkFilterEnumNewest == PkFilterEnumNewest) {
 		checkPackages := []string{}
 		filteredPackageMap := make(map[string]string)
 		for index, pkg := range packages {
@@ -788,13 +792,13 @@ func (bus *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 		}
 
 		// we _could_ do a second resolve and then parse like this...
-		//resolved, e := bus.ResolvePackages(..., filter+PK_FILTER_ENUM_NEWEST)
+		//resolved, e := bus.ResolvePackages(..., filter+PkFilterEnumNewest)
 		// but that's basically what recursion here could do too!
 		if len(checkPackages) > 0 {
-			if PK_DEBUG {
+			if Debug {
 				log.Printf("PackageKit: PackagesToPackageIDs(): Recurse: %v", strings.Join(checkPackages, ", "))
 			}
-			recursion, e = bus.PackagesToPackageIDs(filteredPackageMap, filter+PK_FILTER_ENUM_NEWEST)
+			recursion, e = bus.PackagesToPackageIDs(filteredPackageMap, filter+PkFilterEnumNewest)
 			if e != nil {
 				return nil, fmt.Errorf("Recursion error: %v", e)
 			}