misc: Add mkosi based package building with fpm

Building distro packages is great, however if they aren't built in the
correct environment with the associated dependencies, then they won't
work properly on those distros.

This patch adds an `mkosi` based image building environment that builds
the packages in their respective distros, and then copies them out into
our releases directory.

You'll now want to `make tag && make mkosi && make release` to get a new
release out. We use a small hack to trick the `make release` portion to
not re-build the distro packages if they're already present in the
releases/ directory for that version.

This commit depends on a very recent version of mkosi (it was tested
with git master) and also depends on two currently unmerged patches:
https://github.com/systemd/mkosi/pull/363 and
https://github.com/systemd/mkosi/pull/365

Makefile

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 SHELL = /usr/bin/env bash
-.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag release funcgen
+.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag mkosi mkosi_rpm mkosi_deb mkosi_pacman release releases_path release_rpm release_deb release_pacman funcgen
 .SILENT: clean bindata
 
 # a large amount of output from this `find`, can cause `make` to be much slower!
@@ -165,6 +165,7 @@ clean: ## clean things up
 	$(MAKE) --quiet -C bindata clean
 	$(MAKE) --quiet -C lang/funcs clean
 	$(MAKE) --quiet -C lang clean
+	$(MAKE) --quiet -C misc/mkosi clean
 	rm -f lang/funcs/core/generated_funcs.go || true
 	rm -f lang/funcs/core/generated_funcs_test.go || true
 	[ ! -e $(PROGRAM) ] || rm $(PROGRAM)
@@ -343,11 +344,36 @@ copr: upload-srpms ## build in copr
 tag: ## tags a new release
 	./misc/tag.sh
 
+#
+#	mkosi
+#
+mkosi: mkosi_rpm mkosi_deb mkosi_pacman ## builds distro packages via mkosi
+
+mkosi_rpm: releases/$(VERSION)/.mkdir
+	@echo "Generating: rpm via mkosi..."
+	./misc/mkosi/make.sh rpm mkosi.default.fedora-29 `realpath "releases/$(VERSION)/"`
+
+mkosi_deb: releases/$(VERSION)/.mkdir
+	@echo "Generating: deb via mkosi..."
+	./misc/mkosi/make.sh deb mkosi.default.debian-10 `realpath "releases/$(VERSION)/"`
+
+mkosi_pacman: releases/$(VERSION)/.mkdir
+	@echo "Generating: pacman via mkosi..."
+	./misc/mkosi/make.sh pacman mkosi.default.archlinux `realpath "releases/$(VERSION)/"`
+
 #
 #	release
 #
 release: releases/$(VERSION)/mgmt-release.url ## generates and uploads a release
 
+releases_path:
+	@#Don't put any other output or dependencies in here or they'll show!
+	@echo "releases/$(VERSION)/"
+
+release_rpm: $(RPM_PKG)
+release_deb: $(DEB_PKG)
+release_pacman: $(PACMAN_PKG)
+
 releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA256SUMS_ASC)
 	@echo "Creating github release..."
 	hub release create \
@@ -362,7 +388,7 @@ releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA2
 		|| rm -f releases/$(VERSION)/mgmt-release.url
 
 releases/$(VERSION)/.mkdir:
-	mkdir -p releases/$(VERSION)/{deb,rpm,pacman}/ && touch releases/$(VERSION)/.mkdir
+	mkdir -p releases/$(VERSION)/{rpm,deb,pacman}/ && touch releases/$(VERSION)/.mkdir
 
 releases/$(VERSION)/rpm/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
 	@echo "Generating: rpm changelog..."

misc/fpm-pack.sh

@@ -43,11 +43,37 @@ if [ "$VERSION" != "$TAG" ]; then
 fi
 
 # make sure the package type is valid
-if [ "$1" != "deb" ] && [ "$1" != "rpm" ] && [ "$1" != "pacman" ]; then
+if [ "$1" != "rpm" ] && [ "$1" != "deb" ] && [ "$1" != "pacman" ]; then
 	echo "invalid package type"
 	exit 1
 fi
 
+# don't run if the file already exists (bad idempotent implementation)
+if [ -d "${DIR}/${VERSION}/${1}/" ]; then
+	if [ "$1" = "rpm" ]; then
+		if ls "${DIR}/${VERSION}/${1}/"*.rpm &>/dev/null; then
+			# update timestamp so the Makefile is happy =D
+			touch "${DIR}/${VERSION}/${1}/"*.rpm
+			echo "a .rpm already exists"
+			exit 0	# don't error, we want to be idempotent
+		fi
+	fi
+	if [ "$1" = "deb" ]; then
+		if ls "${DIR}/${VERSION}/${1}/"*.deb &>/dev/null; then
+			touch "${DIR}/${VERSION}/${1}/"*.deb
+			echo "a .deb already exists"
+			exit 0
+		fi
+	fi
+	if [ "$1" = "pacman" ]; then
+		if ls "${DIR}/${VERSION}/${1}/"*.tar.xz &>/dev/null; then
+			touch "${DIR}/${VERSION}/${1}/"*.tar.xz
+			echo "a .tar.xz already exists"
+			exit 0
+		fi
+	fi
+fi
+
 # there are no changelogs for pacman packages
 if [ "$1" != "pacman" ]; then
 	CHANGELOG="--${1}-changelog=${DIR}/${VERSION}/${1}/changelog"
@@ -58,6 +84,11 @@ for i in "${@:3}"; do
 	DEPS="$DEPS -d $i"
 done
 
+# in case the `fpm` gem bin isn't in the $PATH
+if which ruby >/dev/null && which gem >/dev/null && ! command -v fpm 2>/dev/null; then
+	PATH="$(ruby -r rubygems -e 'puts Gem.user_dir')/bin:$PATH"
+fi
+
 # build the package
 fpm \
 	--log error \

misc/make-deps.sh

@@ -45,7 +45,7 @@ if [ ! -z "$YUM" ]; then
 	$sudo_command $YUM install -y ruby-devel rubygems
 	$sudo_command $YUM install -y time
 	# dependencies for building packages with fpm
-	$sudo_command $YUM install -y gcc make rpm-build libffi-devel bsdtar || true
+	$sudo_command $YUM install -y gcc make rpm-build libffi-devel bsdtar mkosi || true
 	$sudo_command $YUM install -y graphviz || true # for debugging
 fi
 if [ ! -z "$APT" ]; then

misc/mkosi/.gitignore

@@ -0,0 +1,5 @@
+mkosi.cache/
+mkosi.builddir/
+mkosi.output/
+image.raw.cache-pre-dev
+image.raw.cache-pre-inst

misc/mkosi/Makefile

@@ -0,0 +1,27 @@
+# Mgmt
+# Copyright (C) 2013-2019+ James Shubin and the project contributors
+# Written by James Shubin <james@shubin.ca> and the project contributors
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+SHELL = /usr/bin/env bash
+.PHONY: all clean
+
+default: all
+
+all:
+	@echo "This Makefile is used for cleaning."
+
+clean:
+	@rm -rf mkosi.{builddir,cache,output}

misc/mkosi/make.sh

@@ -0,0 +1,61 @@
+#!/bin/bash -e
+
+echo running "$0"
+set -o errexit
+set -o pipefail
+
+# Run it the directory this script is in.
+ROOT=$(dirname "${BASH_SOURCE}")
+cd "${ROOT}"
+#pwd
+
+if [ "$3" = "" ]; then
+	# output should be an absolute path
+	echo "Usage: ./$0 <type> <input> <output>"
+	exit 1
+fi
+
+# The type should be one of these.
+if [ "$1" != "rpm" ] && [ "$1" != "deb" ] && [ "$1" != "pacman" ]; then
+	echo "Error: build type sanity check failure."
+	exit 1
+fi
+
+# The input should start with this format string.
+if [[ $2 != mkosi.default.* ]]; then
+	echo "Error: build input sanity check failure."
+	exit 1
+fi
+
+# Make sure we're on a tagged commit.
+TAG=$(git tag -l --points-at HEAD)
+if [ "$TAG" == "" ]; then
+	echo "Error: fpm cannot handle an untagged commit."
+	exit 1
+fi
+
+mkdir -p mkosi.{cache,builddir,output}
+
+# Speed up builds significantly.
+if mountpoint mkosi.output/; then
+	echo "The output directory is already a mountpoint."
+	exit 1
+fi
+echo "Mounting 5G tmpfs in 3 seconds, press ^C to cancel if you are low on RAM."
+sleep 3s
+sudo mount -t tmpfs -o size=5g tmpfs mkosi.output/	# zoom!
+trap 'echo Unmounting tmpfs... && sudo umount mkosi.output/' EXIT	# Unmount on script exit.
+
+echo "Running mkosi (requires root)..."
+time sudo mkosi --default="$2" build	# Test with `summary` instead of `build`.
+
+# FIXME: workaround bug: https://github.com/systemd/mkosi/issues/366
+u=$(id --name --user)
+g=$(id --name --group)
+echo "Running chown (requires root)..."
+sudo chown -R $u:$g mkosi.{cache,builddir}
+
+# Move packaged build artifact into our releases/ directory.
+mv mkosi.builddir/${1}/ "$3"	# mv mkosi.builddir/rpm/ /.../releases/$(VERSION)/
+
+echo "Done $0 run!"

misc/mkosi/mkosi.build

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+# If mkosi.builddir/ exists mkosi will set $BUILDDIR to it, let's then use it as
+# out-of-tree build dir. Otherwise, let's make up our own builddir.
+[ -z "$BUILDDIR" ] && BUILDDIR=build
+
+# After some inspection, we've noticed that:
+# `pwd` is: /root/src
+# $SRCDIR is: /root/src
+# $DESTDIR is: /root/dest
+
+# TODO: Can we use misc/make-path.sh here instead?
+# The default that will get used automatically.
+export GOPATH=~/go/
+
+# The golang bins are stored here.
+export GOBIN="${GOPATH}bin/"
+
+# Needed so that golang build tools will get found.
+export PATH="${GOBIN}:${PATH}"
+
+# Define the golang dir for mgmt.
+MGMTDIR="${GOPATH}src/github.com/purpleidea/mgmt/"
+
+# Make the parent dir.
+mkdir -p $(dirname $MGMTDIR)
+
+# Move our code into it so that golang works.
+mv "$SRCDIR" "$MGMTDIR"
+
+# Work from that directory.
+cd "$MGMTDIR"
+
+# Arch mirror fixes.
+if [ -e "/etc/arch-release" ]; then
+	cp /etc/pacman.d/mirrorlist /etc/pacman.d/mirrorlist.backup
+	# Enable all mirrors.
+	sed -i 's/^#Server/Server/' /etc/pacman.d/mirrorlist.backup
+	#rankmirrors -n 6 /etc/pacman.d/mirrorlist.backup > /etc/pacman.d/mirrorlist	# SLOW
+	echo 'Server = http://mirror.rackspace.com/archlinux/$repo/os/$arch' > /etc/pacman.d/mirrorlist
+	cat /etc/pacman.d/mirrorlist.backup >> /etc/pacman.d/mirrorlist
+	pacman -Syu --noconfirm pacman-mirrorlist
+	pacman -Syyu
+fi
+
+# Get all the dependencies for mgmt.
+make deps
+
+# Build it!
+make || exit 1
+
+# Store the output.
+# TODO: is this required?
+mv mgmt $BUILDDIR
+
+# Get the releases directory path.
+releases_path=$(make releases_path)	# It captures any other output, careful!
+if [ "$releases_path" = "" ]; then
+	echo "Releases path is unknown."
+	exit 1
+fi
+mkdir -p "$releases_path"	# Ensure it exists.
+
+echo "The releases_path is: ${releases_path}."
+
+# Build the package for the distribution that we're in.
+if [ -e "/etc/redhat-release" ]; then
+	# TODO: differentiate between Fedora and CentOS
+	make release_rpm
+	typ='rpm'
+elif [ -e "/etc/debian_version" ]; then
+	# TODO: differentiate between Debian and Ubuntu
+	make release_deb
+	typ='deb'
+elif [ -e "/etc/arch-release" ]; then
+	make release_pacman
+	typ='pacman'
+else
+	echo "OS family is unknown."
+	exit 1
+fi
+echo "The release type is: ${typ}."
+
+# Store releases.
+mv "$releases_path${typ}/" $BUILDDIR	# mv releases/$(VERSION)/rpm/ $BUILDDIR
+
+echo "Done mkosi build!"

misc/mkosi/mkosi.default.archlinux

@@ -0,0 +1,53 @@
+[Distribution]
+Distribution=arch
+#Mirror=http://mirror.rackspace.com/archlinux/
+
+[Output]
+Format=raw_btrfs
+Bootable=yes
+
+# This can be a ramdisk to improve performance.
+OutputDirectory=mkosi.output
+
+[Partitions]
+# 1G isn't enough for the base distros.
+RootSize=5G
+
+[Packages]
+# We need to download sources from the internet.
+WithNetwork=true
+
+# The packages to appear in both the build and the final image.
+Packages=
+	pacman
+	pacman-contrib
+	pacman-mirrorlist
+	make
+	git
+	go
+	tree
+# The packages to appear in the build image, but absent from the final image.
+#BuildPackages=
+#	pacman
+#	pacman-contrib
+#	pacman-mirrorlist
+#	make
+#	git
+#	go
+#	tree
+
+# Store our build artifacts here. This makes them accessible to the outside.
+BuildDirectory=mkosi.builddir
+
+# Cache some of our package manager downloads.
+Cache=mkosi.cache
+
+# Pull the parent git repository into the image.
+BuildSources=../..
+
+# Use this transfer mode for the git sources.
+# XXX: Does auto-detect even work if BuildSources is not the current dir? Bug?
+SourceFileTransfer=copy-git-cached
+
+# Skip the (second) final build phase, since we only need the build artifact.
+SkipFinalPhase=true

misc/mkosi/mkosi.default.debian-10

@@ -0,0 +1,49 @@
+[Distribution]
+Distribution=debian
+Release=stable
+
+[Output]
+Format=raw_btrfs
+Bootable=yes
+
+# This can be a ramdisk to improve performance.
+OutputDirectory=mkosi.output
+
+[Partitions]
+# 1G isn't enough for the base distros.
+RootSize=5G
+
+[Packages]
+# We need to download sources from the internet.
+WithNetwork=true
+
+# The packages to appear in both the build and the final image.
+Packages=
+	apt
+	make
+	git
+	golang
+	tree
+# The packages to appear in the build image, but absent from the final image.
+#BuildPackages=
+#	apt
+#	make
+#	git
+#	golang
+#	tree
+
+# Store our build artifacts here. This makes them accessible to the outside.
+BuildDirectory=mkosi.builddir
+
+# Cache some of our package manager downloads.
+Cache=mkosi.cache
+
+# Pull the parent git repository into the image.
+BuildSources=../..
+
+# Use this transfer mode for the git sources.
+# XXX: Does auto-detect even work if BuildSources is not the current dir? Bug?
+SourceFileTransfer=copy-git-cached
+
+# Skip the (second) final build phase, since we only need the build artifact.
+SkipFinalPhase=true

misc/mkosi/mkosi.default.fedora-29

@@ -0,0 +1,50 @@
+[Distribution]
+Distribution=fedora
+Release=29
+
+[Output]
+Format=raw_btrfs
+#Format=gpt_btrfs
+Bootable=yes
+
+# This can be a ramdisk to improve performance.
+OutputDirectory=mkosi.output
+
+[Partitions]
+# 1G isn't enough for the base distros.
+RootSize=5G
+
+[Packages]
+# We need to download sources from the internet.
+WithNetwork=true
+
+# The packages to appear in both the build and the final image.
+Packages=
+	dnf
+	make
+	git
+	golang
+	tree
+# The packages to appear in the build image, but absent from the final image.
+#BuildPackages=
+#	dnf
+#	make
+#	git
+#	golang
+#	tree
+
+# Store our build artifacts here. This makes them accessible to the outside.
+BuildDirectory=mkosi.builddir
+
+# Cache some of our package manager downloads.
+Cache=mkosi.cache
+
+# Pull the parent git repository into the image.
+BuildSources=../..
+
+# Use this transfer mode for the git sources.
+# XXX: Does auto-detect even work if BuildSources is not the current dir? Bug?
+SourceFileTransfer=copy-git-cached
+
+# Skip the (second) final build phase, since we only need the build artifact.
+SkipFinalPhase=true