misc: Make mkosi building suitable for different distro versions

We'd like to be able to build both Fedora N and N-1 at the same time if possible. This makes it more generally applicable for this scenario, as well as for other distros.
2019-09-20 14:45:21 -04:00
parent a4db9fc8e5
commit 36d2a0de1e
6 changed files with 140 additions and 73 deletions
--- a/54
+++ b/54
@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

 SHELL = /usr/bin/env bash
-.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag mkosi mkosi_rpm mkosi_deb mkosi_pacman release releases_path release_rpm release_deb release_pacman funcgen
+.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag mkosi mkosi_fedora-29 mkosi_debian-10 mkosi_rpm mkosi_deb mkosi_pacman mkosi_archlinux release releases_path release_fedora-29 release_debian-10 release_rpm release_deb release_pacman release_archlinux funcgen
 .SILENT: clean bindata

 # a large amount of output from this `find`, can cause `make` to be much slower!
@@ -49,9 +49,12 @@ GOOSARCHES ?= linux/amd64 linux/ppc64 linux/ppc64le linux/arm64 darwin/amd64
 GOHOSTOS = $(shell go env GOHOSTOS)
 GOHOSTARCH = $(shell go env GOHOSTARCH)

+PKG_FEDORA-29 = releases/$(VERSION)/fedora-29/mgmt-$(VERSION)-1.x86_64.rpm
+PKG_DEBIAN-10 = releases/$(VERSION)/debian-10/mgmt_$(VERSION)_amd64.deb
 RPM_PKG = releases/$(VERSION)/rpm/mgmt-$(VERSION)-1.x86_64.rpm
 DEB_PKG = releases/$(VERSION)/deb/mgmt_$(VERSION)_amd64.deb
 PACMAN_PKG = releases/$(VERSION)/pacman/mgmt-$(VERSION)-1-x86_64.pkg.tar.xz
+PKG_ARCHLINUX = releases/$(VERSION)/archlinux/mgmt-$(VERSION)-1-x86_64.pkg.tar.xz

 SHA256SUMS = releases/$(VERSION)/SHA256SUMS
 SHA256SUMS_ASC = $(SHA256SUMS).asc
@@ -347,7 +350,15 @@ tag: ## tags a new release
 #
 #	mkosi
 #
-mkosi: mkosi_rpm mkosi_deb mkosi_pacman ## builds distro packages via mkosi
+mkosi: mkosi_fedora-29 mkosi_debian-10 mkosi_rpm mkosi_deb mkosi_pacman mkosi_archlinux ## builds distro packages via mkosi
+
+mkosi_fedora-29: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+mkosi_debian-10: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`

 mkosi_rpm: releases/$(VERSION)/.mkdir
 	@echo "Generating: rpm via mkosi..."
@@ -361,6 +372,11 @@ mkosi_pacman: releases/$(VERSION)/.mkdir
 	@echo "Generating: pacman via mkosi..."
 	./misc/mkosi/make.sh pacman mkosi.default.archlinux `realpath "releases/$(VERSION)/"`

+mkosi_archlinux: releases/$(VERSION)/.mkdir
+	@title='$@' ; echo "Generating: $${title#'mkosi_'} via mkosi..."
+	@title='$@' ; distro=$${title#'mkosi_'} ; ./misc/mkosi/make.sh $${distro} `realpath "releases/$(VERSION)/"`
+
+
 #
 #	release
 #
@@ -370,17 +386,23 @@ releases_path:
 	@#Don't put any other output or dependencies in here or they'll show!
 	@echo "releases/$(VERSION)/"

+release_fedora-29: $(PKG_FEDORA-29)
+release_debian-10: $(PKG_DEBIAN-10)
 release_rpm: $(RPM_PKG)
 release_deb: $(DEB_PKG)
 release_pacman: $(PACMAN_PKG)
+release_archlinux: $(PKG_ARCHLINUX)

-releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA256SUMS_ASC)
+releases/$(VERSION)/mgmt-release.url: $(PKG_FEDORA-29) $(PKG_DEBIAN-10) $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(PKG_ARCHLINUX) $(SHA256SUMS_ASC)
 	@echo "Creating github release..."
 	hub release create \
 		-F <( echo -e "$(VERSION)\n";echo "Verify the signatures of all packages before you use them. The signing key can be downloaded from https://purpleidea.com/contact/#pgp-key to verify the release." ) \
+		-a $(PKG_FEDORA-29) \
+		-a $(PKG_DEBIAN-10) \
 		-a $(RPM_PKG) \
 		-a $(DEB_PKG) \
 		-a $(PACMAN_PKG) \
+		-a $(PKG_ARCHLINUX) \
 		-a $(SHA256SUMS_ASC) \
 		$(VERSION) \
 		> releases/$(VERSION)/mgmt-release.url \
@@ -388,7 +410,23 @@ releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA2
 		|| rm -f releases/$(VERSION)/mgmt-release.url

 releases/$(VERSION)/.mkdir:
-	mkdir -p releases/$(VERSION)/{rpm,deb,pacman}/ && touch releases/$(VERSION)/.mkdir
+	mkdir -p releases/$(VERSION)/{fedora-29,debian-10,rpm,deb,pacman,archlinux}/ && touch releases/$(VERSION)/.mkdir
+
+releases/$(VERSION)/fedora-29/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-rpm-changelog.sh "$${distro}" $(VERSION)
+
+$(PKG_FEDORA-29): releases/$(VERSION)/fedora-29/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) libvirt-devel augeas-devel
+
+releases/$(VERSION)/debian-10/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Generating: $${distro} changelog..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/make-deb-changelog.sh "$${distro}" $(VERSION)
+
+$(PKG_DEBIAN-10): releases/$(VERSION)/debian-10/changelog
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) libvirt-dev libaugeas-dev

 releases/$(VERSION)/rpm/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir
 	@echo "Generating: rpm changelog..."
@@ -410,10 +448,14 @@ $(PACMAN_PKG): $(PROGRAM) releases/$(VERSION)/.mkdir
 	@echo "Building: pacman package..."
 	./misc/fpm-pack.sh pacman $(VERSION) libvirt augeas

-$(SHA256SUMS): $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG)
+$(PKG_ARCHLINUX): $(PROGRAM) releases/$(VERSION)/.mkdir
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; echo "Building: $${distro} package..."
+	@title='$(@D)' ; distro=$${title#'releases/$(VERSION)/'} ; ./misc/fpm-pack.sh $${distro} $(VERSION) libvirt augeas
+
+$(SHA256SUMS): $(PKG_FEDORA-29) $(PKG_DEBIAN-10) $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(PKG_ARCHLINUX)
 	@# remove the directory separator in the SHA256SUMS file
 	@echo "Generating: sha256 sum..."
-	sha256sum $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) | awk -F '/| ' '{print $$1"  "$$6}' > $(SHA256SUMS)
+	sha256sum $(PKG_FEDORA-29) $(PKG_DEBIAN-10) $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(PKG_ARCHLINUX) | awk -F '/| ' '{print $$1"  "$$6}' > $(SHA256SUMS)

 $(SHA256SUMS_ASC): $(SHA256SUMS)
 	@echo "Signing sha256 sum..."
--- a/misc/fpm-pack.sh
+++ b/misc/fpm-pack.sh
@@ -1,7 +1,8 @@
 #!/bin/bash
 # This script packages rpm, deb, and pacman packages of mgmt with fpm. The
-# first argument is the package type, and all subsequent arguments are the
-# dependencies. Example usage: `./fpm-pack.sh deb dependency1 dependency2`
+# first argument is the distro type, and the second argument is the version. All
+# subsequent arguments are the dependencies.
+# Example usage: `./fpm-pack.sh fedora-29 0.1.2 dependency1 dependency2`

 # the binary to package
 BINARY="mgmt"
@@ -31,52 +32,62 @@ if [ "$TAG" == "" ]; then
 	exit 1
 fi

-if [ "$2" == "" ]; then
-	echo "version was not specified"
+DISTRO="$1"
+if [ "$1" == "" ]; then
+	echo "distro was not specified"
 	exit 1
 fi
 VERSION="$2"
+if [ "$VERSION" == "" ]; then
+	echo "version was not specified"
+	exit 1
+fi

 if [ "$VERSION" != "$TAG" ]; then
 	echo "you must checkout the correct version before building (${VERSION} != ${TAG})"
 	exit 1
 fi

-# make sure the package type is valid
-if [ "$1" != "rpm" ] && [ "$1" != "deb" ] && [ "$1" != "pacman" ]; then
+# make sure the distro is a known valid one
+if [[ "$DISTRO" == fedora-* ]]; then
+	typ="rpm"
+elif [[ "$DISTRO" == centos-* ]]; then
+	typ="rpm"
+elif [[ "$DISTRO" == debian-* ]]; then
+	typ="deb"
+elif [[ "$DISTRO" == ubuntu-* ]]; then
+	typ="deb"
+elif [[ "$DISTRO" == archlinux ]]; then
+	typ="pacman"
+else
+	echo "unknown distro: ${DISTRO}."
+	exit 1
+fi
+
+if [ "$typ" != "rpm" ] && [ "$typ" != "deb" ] && [ "$typ" != "pacman" ]; then
 	echo "invalid package type"
 	exit 1
 fi

+# assume the file extension
+ext="$typ"
+if [ "$typ" = "pacman" ]; then	# archlinux is an exception
+	ext=".tar.xz"
+fi
+
 # don't run if the file already exists (bad idempotent implementation)
-if [ -d "${DIR}/${VERSION}/${1}/" ]; then
-	if [ "$1" = "rpm" ]; then
-		if ls "${DIR}/${VERSION}/${1}/"*.rpm &>/dev/null; then
-			# update timestamp so the Makefile is happy =D
-			touch "${DIR}/${VERSION}/${1}/"*.rpm
-			echo "a .rpm already exists"
-			exit 0	# don't error, we want to be idempotent
-		fi
-	fi
-	if [ "$1" = "deb" ]; then
-		if ls "${DIR}/${VERSION}/${1}/"*.deb &>/dev/null; then
-			touch "${DIR}/${VERSION}/${1}/"*.deb
-			echo "a .deb already exists"
-			exit 0
-		fi
-	fi
-	if [ "$1" = "pacman" ]; then
-		if ls "${DIR}/${VERSION}/${1}/"*.tar.xz &>/dev/null; then
-			touch "${DIR}/${VERSION}/${1}/"*.tar.xz
-			echo "a .tar.xz already exists"
-			exit 0
-		fi
+if [ -d "${DIR}/${VERSION}/${DISTRO}/" ]; then
+	if ls "${DIR}/${VERSION}/${DISTRO}/"*."${ext}" &>/dev/null; then
+		# update timestamp so the Makefile is happy =D
+		touch "${DIR}/${VERSION}/${DISTRO}/"*."${ext}"
+		echo "a .${ext} already exists"
+		exit 0	# don't error, we want to be idempotent
 	fi
 fi

 # there are no changelogs for pacman packages
-if [ "$1" != "pacman" ]; then
-	CHANGELOG="--${1}-changelog=${DIR}/${VERSION}/${1}/changelog"
+if [ "$typ" != "pacman" ]; then
+	CHANGELOG="--${typ}-changelog=${DIR}/${VERSION}/${DISTRO}/changelog"
 fi

 # arguments after the first two are deps
@@ -85,7 +96,7 @@ for i in "${@:3}"; do
 done

 # in case the `fpm` gem bin isn't in the $PATH
-if which ruby >/dev/null && which gem >/dev/null && ! command -v fpm 2>/dev/null; then
+if command -v ruby >/dev/null && command -v gem >/dev/null && ! command -v fpm 2>/dev/null; then
 	PATH="$(ruby -r rubygems -e 'puts Gem.user_dir')/bin:$PATH"
 fi

@@ -99,8 +110,8 @@ fpm \
 	--description "$DESCRIPTION" \
 	--license "$LICENSE" \
 	--input-type dir \
-	--output-type "$1" \
-	--package "${DIR}/${VERSION}/${1}/" \
+	--output-type "$typ" \
+	--package "${DIR}/${VERSION}/${DISTRO}/" \
 	${CHANGELOG} \
 	${DEPS} \
 	--prefix "$PREFIX" \
--- a/misc/make-deb-changelog.sh
+++ b/misc/make-deb-changelog.sh
@@ -2,9 +2,19 @@
 # This script generates a deb changelog from the project's git history.

 # version we're releasing
-VERSION="$1"
+DISTRO="$1"
+VERSION="$2"
+if [ "$VERSION" = "" ]; then
+	echo "usage: ./$0 <distro> <version>"
+	exit 1
+fi
 # path to store the changelog
-CHANGELOG="releases/${VERSION}/deb/changelog"
+CHANGELOG="releases/${VERSION}/${DISTRO}/changelog"
+dir="$(dirname "$CHANGELOG")/"
+if [ ! -d "$dir" ]; then
+	echo "changelog dir ($dir) does not exist"
+	exit 1
+fi
 # input to format flag for git tag
 TAG_FORMAT="-- %(creator) %(creatordate:format:%a, %d %b %Y %H:%M:%S %z) %(refname:lstrip=2)"
 # a list of tags to be parsed in the loop
--- a/misc/make-rpm-changelog.sh
+++ b/misc/make-rpm-changelog.sh
@@ -2,9 +2,19 @@
 # This script generates an rpm changelog from the project's git history.

 # version we're releasing
-VERSION="$1"
+DISTRO="$1"
+VERSION="$2"
+if [ "$VERSION" = "" ]; then
+	echo "usage: ./$0 <distro> <version>"
+	exit 1
+fi
 # path to store the changelog
-CHANGELOG="releases/${VERSION}/rpm/changelog"
+CHANGELOG="releases/${VERSION}/${DISTRO}/changelog"
+dir="$(dirname "$CHANGELOG")/"
+if [ ! -d "$dir" ]; then
+	echo "changelog dir ($dir) does not exist"
+	exit 1
+fi
 # input to format flag for git tag
 TAG_FORMAT="* %(creatordate:format:%a %b %d %Y) %(creator) %(refname:lstrip=2)"
 # a list of tags to be parsed in the loop
--- a/misc/mkosi/make.sh
+++ b/misc/mkosi/make.sh
@@ -9,21 +9,18 @@ ROOT=$(dirname "${BASH_SOURCE}")
 cd "${ROOT}"
 #pwd

-if [ "$3" = "" ]; then
+if [ "$2" = "" ]; then
 	# output should be an absolute path
-	echo "Usage: ./$0 <type> <input> <output>"
+	echo "Usage: ./$0 <distro> <output>"
 	exit 1
 fi
+distro="$1"	# eg: fedora-29
+output="$2"	# an absolute dir path

-# The type should be one of these.
-if [ "$1" != "rpm" ] && [ "$1" != "deb" ] && [ "$1" != "pacman" ]; then
-	echo "Error: build type sanity check failure."
-	exit 1
-fi
-
-# The input should start with this format string.
-if [[ $2 != mkosi.default.* ]]; then
-	echo "Error: build input sanity check failure."
+# Check that the "default" mkosi distro file exists.
+mkosi_default="mkosi.default.${distro}"	# eg: mkosi.default.fedora-29
+if [ ! -e "${mkosi_default}" ]; then
+	echo "Error: mkosi distro file doesn't exist."
 	exit 1
 fi

@@ -47,7 +44,8 @@ sudo mount -t tmpfs -o size=5g tmpfs mkosi.output/	# zoom!
 trap 'echo Unmounting tmpfs... && sudo umount mkosi.output/' EXIT	# Unmount on script exit.

 echo "Running mkosi (requires root)..."
-time sudo mkosi --default="$2" build	# Test with `summary` instead of `build`.
+# Passing env vars doesn't work, so use: https://github.com/systemd/mkosi/pull/367
+time sudo mkosi --default="${mkosi_default}" build	# Test with `summary` instead of `build`.

 # FIXME: workaround bug: https://github.com/systemd/mkosi/issues/366
 u=$(id --name --user)
@@ -56,6 +54,6 @@ echo "Running chown (requires root)..."
 sudo chown -R $u:$g mkosi.{cache,builddir}

 # Move packaged build artifact into our releases/ directory.
-mv mkosi.builddir/${1}/ "$3"	# mv mkosi.builddir/rpm/ /.../releases/$(VERSION)/
+mv mkosi.builddir/${distro}/ "${output}"	# mv mkosi.builddir/fedora-29/ /.../releases/$(VERSION)/

 echo "Done $0 run!"
--- a/misc/mkosi/mkosi.build
+++ b/misc/mkosi/mkosi.build
@@ -31,6 +31,17 @@ mv "$SRCDIR" "$MGMTDIR"
 # Work from that directory.
 cd "$MGMTDIR"

+# Pull from the MKOSI_DEFAULT var: https://github.com/systemd/mkosi/pull/367
+mkosi_default="mkosi.default."	# remove this prefix
+MGMT_MKOSI_DISTRO="${MKOSI_DEFAULT##$mkosi_default}"
+
+# Lookup the type of image build we're doing.
+if [ "${MGMT_MKOSI_DISTRO}" = "" ]; then
+	echo "The MGMT_MKOSI_DISTRO variable is empty."
+	exit 1
+fi
+echo "Build distro: $MGMT_MKOSI_DISTRO"
+
 # Arch mirror fixes.
 if [ -e "/etc/arch-release" ]; then
 	cp /etc/pacman.d/mirrorlist /etc/pacman.d/mirrorlist.backup
@@ -64,24 +75,9 @@ mkdir -p "$releases_path"	# Ensure it exists.
 echo "The releases_path is: ${releases_path}."

 # Build the package for the distribution that we're in.
-if [ -e "/etc/redhat-release" ]; then
-	# TODO: differentiate between Fedora and CentOS
-	make release_rpm
-	typ='rpm'
-elif [ -e "/etc/debian_version" ]; then
-	# TODO: differentiate between Debian and Ubuntu
-	make release_deb
-	typ='deb'
-elif [ -e "/etc/arch-release" ]; then
-	make release_pacman
-	typ='pacman'
-else
-	echo "OS family is unknown."
-	exit 1
-fi
-echo "The release type is: ${typ}."
+make release_${MGMT_MKOSI_DISTRO}

 # Store releases.
-mv "$releases_path${typ}/" $BUILDDIR	# mv releases/$(VERSION)/rpm/ $BUILDDIR
+mv "$releases_path${MGMT_MKOSI_DISTRO}/" $BUILDDIR	# mv releases/$(VERSION)/fedora-29/ $BUILDDIR

 echo "Done mkosi build!"