From a4db9fc8e5e4439e199abe163c00f5c30e7dd8d1 Mon Sep 17 00:00:00 2001 From: James Shubin Date: Sat, 14 Sep 2019 21:39:25 -0400 Subject: [PATCH] misc: Add mkosi based package building with fpm Building distro packages is great, however if they aren't built in the correct environment with the associated dependencies, then they won't work properly on those distros. This patch adds an `mkosi` based image building environment that builds the packages in their respective distros, and then copies them out into our releases directory. You'll now want to `make tag && make mkosi && make release` to get a new release out. We use a small hack to trick the `make release` portion to not re-build the distro packages if they're already present in the releases/ directory for that version. This commit depends on a very recent version of mkosi (it was tested with git master) and also depends on two currently unmerged patches: https://github.com/systemd/mkosi/pull/363 and https://github.com/systemd/mkosi/pull/365 --- Makefile | 30 ++++++++++- misc/fpm-pack.sh | 33 +++++++++++- misc/make-deps.sh | 2 +- misc/mkosi/.gitignore | 5 ++ misc/mkosi/Makefile | 27 ++++++++++ misc/mkosi/make.sh | 61 +++++++++++++++++++++ misc/mkosi/mkosi.build | 87 ++++++++++++++++++++++++++++++ misc/mkosi/mkosi.default.archlinux | 53 ++++++++++++++++++ misc/mkosi/mkosi.default.debian-10 | 49 +++++++++++++++++ misc/mkosi/mkosi.default.fedora-29 | 50 +++++++++++++++++ 10 files changed, 393 insertions(+), 4 deletions(-) create mode 100644 misc/mkosi/.gitignore create mode 100644 misc/mkosi/Makefile create mode 100755 misc/mkosi/make.sh create mode 100755 misc/mkosi/mkosi.build create mode 100644 misc/mkosi/mkosi.default.archlinux create mode 100644 misc/mkosi/mkosi.default.debian-10 create mode 100644 misc/mkosi/mkosi.default.fedora-29 diff --git a/Makefile b/Makefile index a365cdf3..5b8b5804 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ # along with this program. If not, see . SHELL = /usr/bin/env bash -.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag release funcgen +.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag mkosi mkosi_rpm mkosi_deb mkosi_pacman release releases_path release_rpm release_deb release_pacman funcgen .SILENT: clean bindata # a large amount of output from this `find`, can cause `make` to be much slower! @@ -165,6 +165,7 @@ clean: ## clean things up $(MAKE) --quiet -C bindata clean $(MAKE) --quiet -C lang/funcs clean $(MAKE) --quiet -C lang clean + $(MAKE) --quiet -C misc/mkosi clean rm -f lang/funcs/core/generated_funcs.go || true rm -f lang/funcs/core/generated_funcs_test.go || true [ ! -e $(PROGRAM) ] || rm $(PROGRAM) @@ -343,11 +344,36 @@ copr: upload-srpms ## build in copr tag: ## tags a new release ./misc/tag.sh +# +# mkosi +# +mkosi: mkosi_rpm mkosi_deb mkosi_pacman ## builds distro packages via mkosi + +mkosi_rpm: releases/$(VERSION)/.mkdir + @echo "Generating: rpm via mkosi..." + ./misc/mkosi/make.sh rpm mkosi.default.fedora-29 `realpath "releases/$(VERSION)/"` + +mkosi_deb: releases/$(VERSION)/.mkdir + @echo "Generating: deb via mkosi..." + ./misc/mkosi/make.sh deb mkosi.default.debian-10 `realpath "releases/$(VERSION)/"` + +mkosi_pacman: releases/$(VERSION)/.mkdir + @echo "Generating: pacman via mkosi..." + ./misc/mkosi/make.sh pacman mkosi.default.archlinux `realpath "releases/$(VERSION)/"` + # # release # release: releases/$(VERSION)/mgmt-release.url ## generates and uploads a release +releases_path: + @#Don't put any other output or dependencies in here or they'll show! + @echo "releases/$(VERSION)/" + +release_rpm: $(RPM_PKG) +release_deb: $(DEB_PKG) +release_pacman: $(PACMAN_PKG) + releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA256SUMS_ASC) @echo "Creating github release..." hub release create \ @@ -362,7 +388,7 @@ releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA2 || rm -f releases/$(VERSION)/mgmt-release.url releases/$(VERSION)/.mkdir: - mkdir -p releases/$(VERSION)/{deb,rpm,pacman}/ && touch releases/$(VERSION)/.mkdir + mkdir -p releases/$(VERSION)/{rpm,deb,pacman}/ && touch releases/$(VERSION)/.mkdir releases/$(VERSION)/rpm/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir @echo "Generating: rpm changelog..." diff --git a/misc/fpm-pack.sh b/misc/fpm-pack.sh index a4f2d96c..4e2f9236 100755 --- a/misc/fpm-pack.sh +++ b/misc/fpm-pack.sh @@ -43,11 +43,37 @@ if [ "$VERSION" != "$TAG" ]; then fi # make sure the package type is valid -if [ "$1" != "deb" ] && [ "$1" != "rpm" ] && [ "$1" != "pacman" ]; then +if [ "$1" != "rpm" ] && [ "$1" != "deb" ] && [ "$1" != "pacman" ]; then echo "invalid package type" exit 1 fi +# don't run if the file already exists (bad idempotent implementation) +if [ -d "${DIR}/${VERSION}/${1}/" ]; then + if [ "$1" = "rpm" ]; then + if ls "${DIR}/${VERSION}/${1}/"*.rpm &>/dev/null; then + # update timestamp so the Makefile is happy =D + touch "${DIR}/${VERSION}/${1}/"*.rpm + echo "a .rpm already exists" + exit 0 # don't error, we want to be idempotent + fi + fi + if [ "$1" = "deb" ]; then + if ls "${DIR}/${VERSION}/${1}/"*.deb &>/dev/null; then + touch "${DIR}/${VERSION}/${1}/"*.deb + echo "a .deb already exists" + exit 0 + fi + fi + if [ "$1" = "pacman" ]; then + if ls "${DIR}/${VERSION}/${1}/"*.tar.xz &>/dev/null; then + touch "${DIR}/${VERSION}/${1}/"*.tar.xz + echo "a .tar.xz already exists" + exit 0 + fi + fi +fi + # there are no changelogs for pacman packages if [ "$1" != "pacman" ]; then CHANGELOG="--${1}-changelog=${DIR}/${VERSION}/${1}/changelog" @@ -58,6 +84,11 @@ for i in "${@:3}"; do DEPS="$DEPS -d $i" done +# in case the `fpm` gem bin isn't in the $PATH +if which ruby >/dev/null && which gem >/dev/null && ! command -v fpm 2>/dev/null; then + PATH="$(ruby -r rubygems -e 'puts Gem.user_dir')/bin:$PATH" +fi + # build the package fpm \ --log error \ diff --git a/misc/make-deps.sh b/misc/make-deps.sh index b386e7b7..1900d899 100755 --- a/misc/make-deps.sh +++ b/misc/make-deps.sh @@ -45,7 +45,7 @@ if [ ! -z "$YUM" ]; then $sudo_command $YUM install -y ruby-devel rubygems $sudo_command $YUM install -y time # dependencies for building packages with fpm - $sudo_command $YUM install -y gcc make rpm-build libffi-devel bsdtar || true + $sudo_command $YUM install -y gcc make rpm-build libffi-devel bsdtar mkosi || true $sudo_command $YUM install -y graphviz || true # for debugging fi if [ ! -z "$APT" ]; then diff --git a/misc/mkosi/.gitignore b/misc/mkosi/.gitignore new file mode 100644 index 00000000..1d1f9458 --- /dev/null +++ b/misc/mkosi/.gitignore @@ -0,0 +1,5 @@ +mkosi.cache/ +mkosi.builddir/ +mkosi.output/ +image.raw.cache-pre-dev +image.raw.cache-pre-inst diff --git a/misc/mkosi/Makefile b/misc/mkosi/Makefile new file mode 100644 index 00000000..ad47e5ea --- /dev/null +++ b/misc/mkosi/Makefile @@ -0,0 +1,27 @@ +# Mgmt +# Copyright (C) 2013-2019+ James Shubin and the project contributors +# Written by James Shubin and the project contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +SHELL = /usr/bin/env bash +.PHONY: all clean + +default: all + +all: + @echo "This Makefile is used for cleaning." + +clean: + @rm -rf mkosi.{builddir,cache,output} diff --git a/misc/mkosi/make.sh b/misc/mkosi/make.sh new file mode 100755 index 00000000..326cf870 --- /dev/null +++ b/misc/mkosi/make.sh @@ -0,0 +1,61 @@ +#!/bin/bash -e + +echo running "$0" +set -o errexit +set -o pipefail + +# Run it the directory this script is in. +ROOT=$(dirname "${BASH_SOURCE}") +cd "${ROOT}" +#pwd + +if [ "$3" = "" ]; then + # output should be an absolute path + echo "Usage: ./$0 " + exit 1 +fi + +# The type should be one of these. +if [ "$1" != "rpm" ] && [ "$1" != "deb" ] && [ "$1" != "pacman" ]; then + echo "Error: build type sanity check failure." + exit 1 +fi + +# The input should start with this format string. +if [[ $2 != mkosi.default.* ]]; then + echo "Error: build input sanity check failure." + exit 1 +fi + +# Make sure we're on a tagged commit. +TAG=$(git tag -l --points-at HEAD) +if [ "$TAG" == "" ]; then + echo "Error: fpm cannot handle an untagged commit." + exit 1 +fi + +mkdir -p mkosi.{cache,builddir,output} + +# Speed up builds significantly. +if mountpoint mkosi.output/; then + echo "The output directory is already a mountpoint." + exit 1 +fi +echo "Mounting 5G tmpfs in 3 seconds, press ^C to cancel if you are low on RAM." +sleep 3s +sudo mount -t tmpfs -o size=5g tmpfs mkosi.output/ # zoom! +trap 'echo Unmounting tmpfs... && sudo umount mkosi.output/' EXIT # Unmount on script exit. + +echo "Running mkosi (requires root)..." +time sudo mkosi --default="$2" build # Test with `summary` instead of `build`. + +# FIXME: workaround bug: https://github.com/systemd/mkosi/issues/366 +u=$(id --name --user) +g=$(id --name --group) +echo "Running chown (requires root)..." +sudo chown -R $u:$g mkosi.{cache,builddir} + +# Move packaged build artifact into our releases/ directory. +mv mkosi.builddir/${1}/ "$3" # mv mkosi.builddir/rpm/ /.../releases/$(VERSION)/ + +echo "Done $0 run!" diff --git a/misc/mkosi/mkosi.build b/misc/mkosi/mkosi.build new file mode 100755 index 00000000..390fe1c9 --- /dev/null +++ b/misc/mkosi/mkosi.build @@ -0,0 +1,87 @@ +#!/bin/bash + +# If mkosi.builddir/ exists mkosi will set $BUILDDIR to it, let's then use it as +# out-of-tree build dir. Otherwise, let's make up our own builddir. +[ -z "$BUILDDIR" ] && BUILDDIR=build + +# After some inspection, we've noticed that: +# `pwd` is: /root/src +# $SRCDIR is: /root/src +# $DESTDIR is: /root/dest + +# TODO: Can we use misc/make-path.sh here instead? +# The default that will get used automatically. +export GOPATH=~/go/ + +# The golang bins are stored here. +export GOBIN="${GOPATH}bin/" + +# Needed so that golang build tools will get found. +export PATH="${GOBIN}:${PATH}" + +# Define the golang dir for mgmt. +MGMTDIR="${GOPATH}src/github.com/purpleidea/mgmt/" + +# Make the parent dir. +mkdir -p $(dirname $MGMTDIR) + +# Move our code into it so that golang works. +mv "$SRCDIR" "$MGMTDIR" + +# Work from that directory. +cd "$MGMTDIR" + +# Arch mirror fixes. +if [ -e "/etc/arch-release" ]; then + cp /etc/pacman.d/mirrorlist /etc/pacman.d/mirrorlist.backup + # Enable all mirrors. + sed -i 's/^#Server/Server/' /etc/pacman.d/mirrorlist.backup + #rankmirrors -n 6 /etc/pacman.d/mirrorlist.backup > /etc/pacman.d/mirrorlist # SLOW + echo 'Server = http://mirror.rackspace.com/archlinux/$repo/os/$arch' > /etc/pacman.d/mirrorlist + cat /etc/pacman.d/mirrorlist.backup >> /etc/pacman.d/mirrorlist + pacman -Syu --noconfirm pacman-mirrorlist + pacman -Syyu +fi + +# Get all the dependencies for mgmt. +make deps + +# Build it! +make || exit 1 + +# Store the output. +# TODO: is this required? +mv mgmt $BUILDDIR + +# Get the releases directory path. +releases_path=$(make releases_path) # It captures any other output, careful! +if [ "$releases_path" = "" ]; then + echo "Releases path is unknown." + exit 1 +fi +mkdir -p "$releases_path" # Ensure it exists. + +echo "The releases_path is: ${releases_path}." + +# Build the package for the distribution that we're in. +if [ -e "/etc/redhat-release" ]; then + # TODO: differentiate between Fedora and CentOS + make release_rpm + typ='rpm' +elif [ -e "/etc/debian_version" ]; then + # TODO: differentiate between Debian and Ubuntu + make release_deb + typ='deb' +elif [ -e "/etc/arch-release" ]; then + make release_pacman + typ='pacman' +else + echo "OS family is unknown." + exit 1 +fi +echo "The release type is: ${typ}." + +# Store releases. +mv "$releases_path${typ}/" $BUILDDIR # mv releases/$(VERSION)/rpm/ $BUILDDIR + +echo "Done mkosi build!" diff --git a/misc/mkosi/mkosi.default.archlinux b/misc/mkosi/mkosi.default.archlinux new file mode 100644 index 00000000..4e56d9c6 --- /dev/null +++ b/misc/mkosi/mkosi.default.archlinux @@ -0,0 +1,53 @@ +[Distribution] +Distribution=arch +#Mirror=http://mirror.rackspace.com/archlinux/ + +[Output] +Format=raw_btrfs +Bootable=yes + +# This can be a ramdisk to improve performance. +OutputDirectory=mkosi.output + +[Partitions] +# 1G isn't enough for the base distros. +RootSize=5G + +[Packages] +# We need to download sources from the internet. +WithNetwork=true + +# The packages to appear in both the build and the final image. +Packages= + pacman + pacman-contrib + pacman-mirrorlist + make + git + go + tree +# The packages to appear in the build image, but absent from the final image. +#BuildPackages= +# pacman +# pacman-contrib +# pacman-mirrorlist +# make +# git +# go +# tree + +# Store our build artifacts here. This makes them accessible to the outside. +BuildDirectory=mkosi.builddir + +# Cache some of our package manager downloads. +Cache=mkosi.cache + +# Pull the parent git repository into the image. +BuildSources=../.. + +# Use this transfer mode for the git sources. +# XXX: Does auto-detect even work if BuildSources is not the current dir? Bug? +SourceFileTransfer=copy-git-cached + +# Skip the (second) final build phase, since we only need the build artifact. +SkipFinalPhase=true diff --git a/misc/mkosi/mkosi.default.debian-10 b/misc/mkosi/mkosi.default.debian-10 new file mode 100644 index 00000000..14fcdf26 --- /dev/null +++ b/misc/mkosi/mkosi.default.debian-10 @@ -0,0 +1,49 @@ +[Distribution] +Distribution=debian +Release=stable + +[Output] +Format=raw_btrfs +Bootable=yes + +# This can be a ramdisk to improve performance. +OutputDirectory=mkosi.output + +[Partitions] +# 1G isn't enough for the base distros. +RootSize=5G + +[Packages] +# We need to download sources from the internet. +WithNetwork=true + +# The packages to appear in both the build and the final image. +Packages= + apt + make + git + golang + tree +# The packages to appear in the build image, but absent from the final image. +#BuildPackages= +# apt +# make +# git +# golang +# tree + +# Store our build artifacts here. This makes them accessible to the outside. +BuildDirectory=mkosi.builddir + +# Cache some of our package manager downloads. +Cache=mkosi.cache + +# Pull the parent git repository into the image. +BuildSources=../.. + +# Use this transfer mode for the git sources. +# XXX: Does auto-detect even work if BuildSources is not the current dir? Bug? +SourceFileTransfer=copy-git-cached + +# Skip the (second) final build phase, since we only need the build artifact. +SkipFinalPhase=true diff --git a/misc/mkosi/mkosi.default.fedora-29 b/misc/mkosi/mkosi.default.fedora-29 new file mode 100644 index 00000000..18c7b3d9 --- /dev/null +++ b/misc/mkosi/mkosi.default.fedora-29 @@ -0,0 +1,50 @@ +[Distribution] +Distribution=fedora +Release=29 + +[Output] +Format=raw_btrfs +#Format=gpt_btrfs +Bootable=yes + +# This can be a ramdisk to improve performance. +OutputDirectory=mkosi.output + +[Partitions] +# 1G isn't enough for the base distros. +RootSize=5G + +[Packages] +# We need to download sources from the internet. +WithNetwork=true + +# The packages to appear in both the build and the final image. +Packages= + dnf + make + git + golang + tree +# The packages to appear in the build image, but absent from the final image. +#BuildPackages= +# dnf +# make +# git +# golang +# tree + +# Store our build artifacts here. This makes them accessible to the outside. +BuildDirectory=mkosi.builddir + +# Cache some of our package manager downloads. +Cache=mkosi.cache + +# Pull the parent git repository into the image. +BuildSources=../.. + +# Use this transfer mode for the git sources. +# XXX: Does auto-detect even work if BuildSources is not the current dir? Bug? +SourceFileTransfer=copy-git-cached + +# Skip the (second) final build phase, since we only need the build artifact. +SkipFinalPhase=true