make: Release pipeline

This commit adds new make targets for rpm, deb, and pacman packages.
It also adds a phony target that uploads tarballs of the packages,
along with their signed (and unsigned) checksums to the github release
page. Once the current commit is tagged as a release, run `make release`
to build the packages and upload them to github.

.gitignore

@@ -13,4 +13,4 @@ mgmt.static
 build/mgmt-*
 mgmt.iml
 rpmbuild/
-*.deb
+releases/

Makefile

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 SHELL = /usr/bin/env bash
-.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms copr
+.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms copr release release/mkdirs
 .SILENT: clean bindata
 
 # a large amount of output from this `find`, can cause `make` to be much slower!
@@ -48,6 +48,10 @@ GOOSARCHES ?= linux/amd64 linux/ppc64 linux/ppc64le linux/arm64 darwin/amd64
 GOHOSTOS = $(shell go env GOHOSTOS)
 GOHOSTARCH = $(shell go env GOHOSTARCH)
 
+DEB_TAR = releases/mgmt_$(VERSION)_deb_amd64.tar.gz
+RPM_TAR = releases/mgmt_$(VERSION)_rpm_x86_64.tar.gz
+PACMAN_TAR = releases/mgmt_$(VERSION)_pacman_x86_64.tar.gz
+
 default: build
 
 #
@@ -324,16 +328,81 @@ copr: upload-srpms ## build in copr
 	./misc/copr-build.py https://$(SERVER)/$(REMOTE_PATH)/SRPMS/$(SRPM_BASE)
 
 #
-#	deb build
+#	release
 #
 
-deb: ## build debian package
+release: releases/mgmt-$(VERSION)-release.url
-	./misc/gen-deb-changelog-from-git.sh
+
-	dpkg-buildpackage
+releases/mgmt-$(VERSION)-release.url: $(DEB_TAR) $(RPM_TAR) $(PACMAN_TAR)
-	# especially when building in Docker container, pull build artifact in project directory.
+	@echo "Creating github release..."
-	cp ../mgmt_*_amd64.deb ./
+	hub release create \
-	# cleanup
+		-F <( echo -e "$(VERSION)\n";echo "License: GPLv3" ) \
-	rm -rf debian/mgmt/
+		-a $(DEB_TAR) \
+		-a $(RPM_TAR) \
+		-a $(PACMAN_TAR) \
+		$(VERSION) \
+		> releases/mgmt-$(VERSION)-release.url \
+		&& cat releases/mgmt-$(VERSION)-release.url \
+		|| rm -f releases/mgmt-$(VERSION)-release.url
+
+release/mkdirs:
+	mkdir -p releases/{deb,rpm,pacman}
+
+$(DEB_TAR): releases/deb/SHA256SUMS.asc
+	@echo -e "Archiving deb package..."
+	tar -zcf $(DEB_TAR) -C releases/deb .
+
+releases/deb/SHA256SUMS.asc: releases/deb/SHA256SUMS
+	@echo "Signing sha256 sum..."
+	gpg2 --yes --clearsign releases/deb/SHA256SUMS
+
+releases/deb/SHA256SUMS: releases/deb/mgmt_$(VERSION)_amd64.deb
+	@echo "Generating sha256 sum..."
+	sha256sum releases/deb/*.deb > releases/deb/SHA256SUMS
+
+releases/deb/mgmt_$(VERSION)_amd64.deb: releases/deb/changelog
+	@echo "Building deb package..."
+	./misc/fpm-pack.sh deb libvirt-dev libaugeas-dev
+
+releases/deb/changelog: $(PROGRAM) | release/mkdirs
+	@echo "Generating deb changelog..."
+	./misc/make-deb-changelog.sh
+
+$(RPM_TAR): releases/rpm/SHA256SUMS.asc
+	@echo -e "Archiving rpm package..."
+	tar -zcf $(RPM_TAR) -C releases/rpm .
+
+releases/rpm/SHA256SUMS.asc: releases/rpm/SHA256SUMS
+	@echo "Signing sha256 sum..."
+	gpg2 --yes --clearsign releases/rpm/SHA256SUMS
+
+releases/rpm/SHA256SUMS: releases/rpm/mgmt-$(VERSION)-1.x86_64.rpm
+	@echo "Generating sha256 sum..."
+	sha256sum releases/rpm/*.rpm > releases/rpm/SHA256SUMS
+
+releases/rpm/mgmt-$(VERSION)-1.x86_64.rpm: releases/rpm/changelog
+	@echo "Building rpm package..."
+	./misc/fpm-pack.sh rpm libvirt-devel augeas-devel
+
+releases/rpm/changelog: $(PROGRAM) | release/mkdirs
+	@echo "Generating rpm changelog..."
+	./misc/make-rpm-changelog.sh
+
+$(PACMAN_TAR): releases/pacman/SHA256SUMS.asc
+	@echo -e "Archiving pacman package..."
+	tar -zcf $(PACMAN_TAR) -C releases/pacman .
+
+releases/pacman/SHA256SUMS.asc: releases/pacman/SHA256SUMS
+	@echo "Signing sha256 sum..."
+	gpg2 --yes --clearsign releases/pacman/SHA256SUMS
+
+releases/pacman/SHA256SUMS: releases/pacman/mgmt-$(VERSION)-1-x86_64.pkg.tar.xz
+	@echo "Generating sha256 sum..."
+	sha256sum releases/pacman/*.pkg.tar.xz > releases/pacman/SHA256SUMS
+
+releases/pacman/mgmt-$(VERSION)-1-x86_64.pkg.tar.xz: $(PROGRAM) | release/mkdirs
+	@echo "Building pacman package..."
+	./misc/fpm-pack.sh pacman libvirt augeas
 
 build_container: ## builds the container
 	docker build -t purpleidea/mgmt-build -f docker/Dockerfile.build .

misc/fpm-pack.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# This script packages rpm, deb, and pacman packages of mgmt with fpm. The
+# first argument is the package type, and all subsequent arguments are the
+# dependencies. Example usage: `./fpm-pack.sh deb dependency1 dependency2`
+
+# the binary to package
+BINARY="mgmt"
+# git tag pointing to the current commit
+TAG=$(git tag -l --points-at HEAD)
+# maintainer email
+MAINTAINER="mgmt@noreply.github.com"
+# project url
+URL="https://github.com/purpleidea/mgmt/"
+# project description
+DESCRIPTION="Next generation distributed, event-driven, parallel config management!"
+# project license
+LICENSE="GPLv3"
+# location to install the binary
+PREFIX="/usr/bin"
+# release directory
+DIR="releases"
+
+# placeholder for dependencies to be read from arguments
+DEPS=
+# placeholder for changelog argument parsed from the package type
+CHANGELOG=
+
+# make sure we're on a tagged commit
+if [ "$TAG" == "" ]; then
+	echo "cannot release an untagged commit"
+	exit 1
+fi
+
+# make sure the package type is valid
+if [ "$1" != "deb" ] && [ "$1" != "rpm" ] && [ "$1" != "pacman" ]; then
+	echo "invalid package type"
+	exit 1
+fi
+
+# there are no changelogs for pacman packages
+if [ "$1" != "pacman" ]; then
+	CHANGELOG="--${1}-changelog=${DIR}/${1}/changelog"
+fi
+
+# arguments after the first one are deps
+for i in "${@:2}"; do
+	DEPS="$DEPS -d $i"
+done
+
+# build the package
+fpm \
+	--log error \
+	--name "$BINARY" \
+	--version "$TAG" \
+	--maintainer "$MAINTAINER" \
+	--url "$URL" \
+	--description "$DESCRIPTION" \
+	--license "$LICENSE" \
+	--input-type dir \
+	--output-type "$1" \
+	--package "${DIR}/${1}/" \
+	${CHANGELOG} \
+	${DEPS} \
+	--prefix "$PREFIX" \
+	"$BINARY"

misc/gen-deb-changelog-from-git.sh

@@ -1,76 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-IFS=$'\n\t'
-
-PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
-
-cleanup() {
-	if [ -f "${tmpfile}" ]; then
-		rm -f "${tmpfile}"
-	fi
-}
-
-trap "{ cleanup; }" EXIT SIGTERM
-
-getCommits() {
-	prevtag="${1}"
-	tag="${2}"
-	local -a authors
-	local ver="${tag}-1"
-	local h
-
-	echo "»»» Processing ${prevtag}..${tag}"
-	numCommits=$(git --no-pager rev-list --count "${prevtag}".."${tag}")
-	if ((numCommits>0)); then
-		echo "	${numCommits} commits found"
-
-		if [ "${tag}" == "HEAD" ]; then
-			h=$(git rev-list --max-count=1 --abbrev-commit HEAD)
-			ver="${prevtag}~1.${h}"
-		fi
-
-		echo "${pkgname} (${ver}) UNRELEASED; urgency=low" >> "${tmpfile}"
-
-		authors=($(git log --format='%aN' "${prevtag}".."${tag}" | sort | uniq))
-		for author in "${authors[@]}"; do
-			echo "	Gathering commits from ${author}"
-			{
-				echo "	[ ${author} ]"
-				git --no-pager log --author="${author}" --pretty=format:'  * %s' "${prevtag}".."${tag}"
-				echo ""
-			} >> "${tmpfile}"
-		done
-
-		git --no-pager log -n 1 --pretty='format:%n -- %aN <%aE>  %aD%n%n' "${tag}" >> "${tmpfile}"
-	else
-		echo "	0 commits found, skipping"
-	fi
-}
-
-if [ ! -d "debian" ]; then
-	echo "Directory ./debian not found"
-	exit 1
-fi
-
-tmpfile=$(mktemp)
-firstHash=$(git rev-list --max-parents=0 HEAD) # This should yield the very first commit hash
-pkgname=$(grep '^Package: ' debian/control | sed 's/^Package: //')
-tags=($(git tag | sort -r -V))
-
-echo "»»» Gathering untagged commits"
-tag=${tags[0]}
-untagged=$(git rev-list --count "${tag}"..HEAD)
-if ((untagged>0)); then
-	getCommits "${tag}" HEAD
-fi
-
-
-for ((i=1; i<${#tags[@]}; i++)); do
-	tag="${tags[${i}]}"
-	nexttag="${tags[$((i-1))]}"
-	getCommits "${tag}" "${nexttag}"
-done
-
-getCommits "${firstHash}" "${tags[-1]}"
-
-mv "${tmpfile}" debian/changelog

misc/make-deb-changelog.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# This script generates a deb changelog from the project's git history.
+
+# path to store the changelog
+CHANGELOG="releases/deb/changelog"
+# input to format flag for git tag
+TAG_FORMAT="-- %(creator) %(creatordate:format:%a, %d %b %Y %H:%M:%S %z) %(refname:lstrip=2)"
+# a list of tags to be parsed in the loop
+TAGS=$(git tag --sort=-creatordate --format="$TAG_FORMAT" | sed -r 's/[0-9]+ -[0-9]+ //')
+
+# placeholder for the next line of the list
+THIS_TAGLINE=
+
+# parse the list
+while read -r LAST_TAGLINE; do
+	# read ahead one tag
+	if [ "$THIS_TAGLINE" == "" ]; then
+		# store the tag for the next iteration
+		THIS_TAGLINE="$LAST_TAGLINE"
+		continue
+	fi
+
+	# grab the tags from the last column of the taglines
+	THIS_TAG=$(echo "$THIS_TAGLINE" | awk '{print $NF}')
+	LAST_TAG=$(echo "$LAST_TAGLINE" | awk '{print $NF}')
+
+	# print the release description
+	printf "mgmt (%s) unstable; priority=low\n\n" "$THIS_TAG" >> "$CHANGELOG"
+
+	# print all the commits between the tags
+	git shortlog -n "${LAST_TAG}...${THIS_TAG}" | sed -r '/\):/s/^/ * /' >> "$CHANGELOG"
+
+	# print the release signature
+	printf "%s\n\n\n" "$THIS_TAGLINE" | sed -r 's/[0-9]\.[0-9]\.[0-9]+//'>> "$CHANGELOG"
+
+	# first tag is special since there's no previous one
+	if [ "$LAST_TAG" == "0.0.1" ]; then
+		# print all the commits before the first tag
+		git shortlog -n "$LAST_TAG" | sed -r '/\):/s/^/ * /' >> "$CHANGELOG"
+		# print the release signature
+		printf "%s\n" "$LAST_TAGLINE" | sed -r 's/[0-9]\.[0-9]\.[0-9]+//'>> "$CHANGELOG"
+	fi
+
+	# store the tag for the next iteration
+	THIS_TAGLINE="$LAST_TAGLINE"
+done <<< "$TAGS"

misc/make-deps.sh

@@ -31,16 +31,18 @@ fi
 if [ ! -z "$YUM" ]; then
 	$sudo_command $YUM install -y libvirt-devel
 	$sudo_command $YUM install -y augeas-devel
-	$sudo_command $YUM install -y rubygems
+	$sudo_command $YUM install -y ruby-devel rubygems
 	$sudo_command $YUM install -y time
+	# dependencies for building packages with fpm
+	$sudo_command $YUM install -y gcc make rpm-build libffi-devel bsdtar || true
 fi
 if [ ! -z "$APT" ]; then
 	$sudo_command $APT install -y libvirt-dev || true
 	$sudo_command $APT install -y libaugeas-dev || true
-	$sudo_command $APT install -y rubygems || true
+	$sudo_command $APT install -y ruby ruby-dev rubygems || true
 	$sudo_command $APT install -y libpcap0.8-dev || true
-	# dependencies for building debian packages with `make deb`
+	# dependencies for building packages with fpm
-	$sudo_command $APT install -y dpkg-dev devscripts debhelper dh-golang dh-systemd
+	$sudo_command $APT install -y build-essential rpm bsdtar || true
 	# `realpath` is a more universal alternative to `readlink -f` for absolute path resolution
 	# (-f is missing on BSD/macOS), but older Debian/Ubuntu's don't include it in coreutils yet.
 	# https://unix.stackexchange.com/a/136527
@@ -103,4 +105,5 @@ go get golang.org/x/lint/golint				# for `golint`-ing
 go get github.com/tmthrgd/go-bindata/go-bindata	# for compiling in non golang files
 go get -u gopkg.in/alecthomas/gometalinter.v1 && mv "$(dirname $(command -v gometalinter.v1))/gometalinter.v1" "$(dirname $(command -v gometalinter.v1))/gometalinter" && gometalinter --install	# bonus
 command -v mdl &>/dev/null || gem install mdl || true	# for linting markdown files
+command -v fpm &>/dev/null || gem install fpm || true	# for cross distro packaging
 cd "$XPWD" >/dev/null

misc/make-rpm-changelog.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# This script generates an rpm changelog from the project's git history.
+
+# path to store the changelog
+CHANGELOG="releases/rpm/changelog"
+# input to format flag for git tag
+TAG_FORMAT="* %(creatordate:format:%a %b %d %Y) %(creator) %(refname:lstrip=2)"
+# a list of tags to be parsed in the loop
+TAGS=$(git tag --sort=-creatordate --format="$TAG_FORMAT" | sed -r 's/[0-9]+ -[0-9]+ //')
+
+# placeholder for the next line of the list
+THIS_TAGLINE=
+
+# parse the list
+while read -r LAST_TAGLINE; do
+	# read ahead one tag
+	if [ "$THIS_TAGLINE" == "" ]; then
+		# store the tag for the next iteration
+		THIS_TAGLINE="$LAST_TAGLINE"
+		continue
+	fi
+
+	# grab the tags from the last column of the taglines
+	THIS_TAG=$(echo "$THIS_TAGLINE" | awk '{print $NF}')
+	LAST_TAG=$(echo "$LAST_TAGLINE" | awk '{print $NF}')
+
+	# print the release description
+	printf "\n%s\n\n" "$THIS_TAGLINE" >> "$CHANGELOG"
+
+	# print all the commits between the tags
+	git shortlog -n ${LAST_TAG}...${THIS_TAG} | sed -r '/\):/s/^/ - /' >> "$CHANGELOG"
+
+	# first tag is special since there's no previous one
+	if [ "$LAST_TAG" == "0.0.1" ]; then
+		# print the release description
+		printf "\n%s\n\n" "$LAST_TAGLINE" >> "$CHANGELOG"
+		# print all the commits before the first tag
+		git shortlog -n $LAST_TAG | sed -r '/\):/s/^/ - /' >> "$CHANGELOG"
+	fi
+
+	# store the tag for the next iteration
+	THIS_TAGLINE="$LAST_TAGLINE"
+done <<< "$TAGS"
+
+# trim the first and last lines
+sed -i '1d;$d' "$CHANGELOG"