diff --git a/Makefile b/Makefile index a365cdf3..5b8b5804 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ # along with this program. If not, see . SHELL = /usr/bin/env bash -.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag release funcgen +.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag mkosi mkosi_rpm mkosi_deb mkosi_pacman release releases_path release_rpm release_deb release_pacman funcgen .SILENT: clean bindata # a large amount of output from this `find`, can cause `make` to be much slower! @@ -165,6 +165,7 @@ clean: ## clean things up $(MAKE) --quiet -C bindata clean $(MAKE) --quiet -C lang/funcs clean $(MAKE) --quiet -C lang clean + $(MAKE) --quiet -C misc/mkosi clean rm -f lang/funcs/core/generated_funcs.go || true rm -f lang/funcs/core/generated_funcs_test.go || true [ ! -e $(PROGRAM) ] || rm $(PROGRAM) @@ -343,11 +344,36 @@ copr: upload-srpms ## build in copr tag: ## tags a new release ./misc/tag.sh +# +# mkosi +# +mkosi: mkosi_rpm mkosi_deb mkosi_pacman ## builds distro packages via mkosi + +mkosi_rpm: releases/$(VERSION)/.mkdir + @echo "Generating: rpm via mkosi..." + ./misc/mkosi/make.sh rpm mkosi.default.fedora-29 `realpath "releases/$(VERSION)/"` + +mkosi_deb: releases/$(VERSION)/.mkdir + @echo "Generating: deb via mkosi..." + ./misc/mkosi/make.sh deb mkosi.default.debian-10 `realpath "releases/$(VERSION)/"` + +mkosi_pacman: releases/$(VERSION)/.mkdir + @echo "Generating: pacman via mkosi..." + ./misc/mkosi/make.sh pacman mkosi.default.archlinux `realpath "releases/$(VERSION)/"` + # # release # release: releases/$(VERSION)/mgmt-release.url ## generates and uploads a release +releases_path: + @#Don't put any other output or dependencies in here or they'll show! + @echo "releases/$(VERSION)/" + +release_rpm: $(RPM_PKG) +release_deb: $(DEB_PKG) +release_pacman: $(PACMAN_PKG) + releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA256SUMS_ASC) @echo "Creating github release..." hub release create \ @@ -362,7 +388,7 @@ releases/$(VERSION)/mgmt-release.url: $(RPM_PKG) $(DEB_PKG) $(PACMAN_PKG) $(SHA2 || rm -f releases/$(VERSION)/mgmt-release.url releases/$(VERSION)/.mkdir: - mkdir -p releases/$(VERSION)/{deb,rpm,pacman}/ && touch releases/$(VERSION)/.mkdir + mkdir -p releases/$(VERSION)/{rpm,deb,pacman}/ && touch releases/$(VERSION)/.mkdir releases/$(VERSION)/rpm/changelog: $(PROGRAM) releases/$(VERSION)/.mkdir @echo "Generating: rpm changelog..." diff --git a/misc/fpm-pack.sh b/misc/fpm-pack.sh index a4f2d96c..4e2f9236 100755 --- a/misc/fpm-pack.sh +++ b/misc/fpm-pack.sh @@ -43,11 +43,37 @@ if [ "$VERSION" != "$TAG" ]; then fi # make sure the package type is valid -if [ "$1" != "deb" ] && [ "$1" != "rpm" ] && [ "$1" != "pacman" ]; then +if [ "$1" != "rpm" ] && [ "$1" != "deb" ] && [ "$1" != "pacman" ]; then echo "invalid package type" exit 1 fi +# don't run if the file already exists (bad idempotent implementation) +if [ -d "${DIR}/${VERSION}/${1}/" ]; then + if [ "$1" = "rpm" ]; then + if ls "${DIR}/${VERSION}/${1}/"*.rpm &>/dev/null; then + # update timestamp so the Makefile is happy =D + touch "${DIR}/${VERSION}/${1}/"*.rpm + echo "a .rpm already exists" + exit 0 # don't error, we want to be idempotent + fi + fi + if [ "$1" = "deb" ]; then + if ls "${DIR}/${VERSION}/${1}/"*.deb &>/dev/null; then + touch "${DIR}/${VERSION}/${1}/"*.deb + echo "a .deb already exists" + exit 0 + fi + fi + if [ "$1" = "pacman" ]; then + if ls "${DIR}/${VERSION}/${1}/"*.tar.xz &>/dev/null; then + touch "${DIR}/${VERSION}/${1}/"*.tar.xz + echo "a .tar.xz already exists" + exit 0 + fi + fi +fi + # there are no changelogs for pacman packages if [ "$1" != "pacman" ]; then CHANGELOG="--${1}-changelog=${DIR}/${VERSION}/${1}/changelog" @@ -58,6 +84,11 @@ for i in "${@:3}"; do DEPS="$DEPS -d $i" done +# in case the `fpm` gem bin isn't in the $PATH +if which ruby >/dev/null && which gem >/dev/null && ! command -v fpm 2>/dev/null; then + PATH="$(ruby -r rubygems -e 'puts Gem.user_dir')/bin:$PATH" +fi + # build the package fpm \ --log error \ diff --git a/misc/make-deps.sh b/misc/make-deps.sh index b386e7b7..1900d899 100755 --- a/misc/make-deps.sh +++ b/misc/make-deps.sh @@ -45,7 +45,7 @@ if [ ! -z "$YUM" ]; then $sudo_command $YUM install -y ruby-devel rubygems $sudo_command $YUM install -y time # dependencies for building packages with fpm - $sudo_command $YUM install -y gcc make rpm-build libffi-devel bsdtar || true + $sudo_command $YUM install -y gcc make rpm-build libffi-devel bsdtar mkosi || true $sudo_command $YUM install -y graphviz || true # for debugging fi if [ ! -z "$APT" ]; then diff --git a/misc/mkosi/.gitignore b/misc/mkosi/.gitignore new file mode 100644 index 00000000..1d1f9458 --- /dev/null +++ b/misc/mkosi/.gitignore @@ -0,0 +1,5 @@ +mkosi.cache/ +mkosi.builddir/ +mkosi.output/ +image.raw.cache-pre-dev +image.raw.cache-pre-inst diff --git a/misc/mkosi/Makefile b/misc/mkosi/Makefile new file mode 100644 index 00000000..ad47e5ea --- /dev/null +++ b/misc/mkosi/Makefile @@ -0,0 +1,27 @@ +# Mgmt +# Copyright (C) 2013-2019+ James Shubin and the project contributors +# Written by James Shubin and the project contributors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +SHELL = /usr/bin/env bash +.PHONY: all clean + +default: all + +all: + @echo "This Makefile is used for cleaning." + +clean: + @rm -rf mkosi.{builddir,cache,output} diff --git a/misc/mkosi/make.sh b/misc/mkosi/make.sh new file mode 100755 index 00000000..326cf870 --- /dev/null +++ b/misc/mkosi/make.sh @@ -0,0 +1,61 @@ +#!/bin/bash -e + +echo running "$0" +set -o errexit +set -o pipefail + +# Run it the directory this script is in. +ROOT=$(dirname "${BASH_SOURCE}") +cd "${ROOT}" +#pwd + +if [ "$3" = "" ]; then + # output should be an absolute path + echo "Usage: ./$0 " + exit 1 +fi + +# The type should be one of these. +if [ "$1" != "rpm" ] && [ "$1" != "deb" ] && [ "$1" != "pacman" ]; then + echo "Error: build type sanity check failure." + exit 1 +fi + +# The input should start with this format string. +if [[ $2 != mkosi.default.* ]]; then + echo "Error: build input sanity check failure." + exit 1 +fi + +# Make sure we're on a tagged commit. +TAG=$(git tag -l --points-at HEAD) +if [ "$TAG" == "" ]; then + echo "Error: fpm cannot handle an untagged commit." + exit 1 +fi + +mkdir -p mkosi.{cache,builddir,output} + +# Speed up builds significantly. +if mountpoint mkosi.output/; then + echo "The output directory is already a mountpoint." + exit 1 +fi +echo "Mounting 5G tmpfs in 3 seconds, press ^C to cancel if you are low on RAM." +sleep 3s +sudo mount -t tmpfs -o size=5g tmpfs mkosi.output/ # zoom! +trap 'echo Unmounting tmpfs... && sudo umount mkosi.output/' EXIT # Unmount on script exit. + +echo "Running mkosi (requires root)..." +time sudo mkosi --default="$2" build # Test with `summary` instead of `build`. + +# FIXME: workaround bug: https://github.com/systemd/mkosi/issues/366 +u=$(id --name --user) +g=$(id --name --group) +echo "Running chown (requires root)..." +sudo chown -R $u:$g mkosi.{cache,builddir} + +# Move packaged build artifact into our releases/ directory. +mv mkosi.builddir/${1}/ "$3" # mv mkosi.builddir/rpm/ /.../releases/$(VERSION)/ + +echo "Done $0 run!" diff --git a/misc/mkosi/mkosi.build b/misc/mkosi/mkosi.build new file mode 100755 index 00000000..390fe1c9 --- /dev/null +++ b/misc/mkosi/mkosi.build @@ -0,0 +1,87 @@ +#!/bin/bash + +# If mkosi.builddir/ exists mkosi will set $BUILDDIR to it, let's then use it as +# out-of-tree build dir. Otherwise, let's make up our own builddir. +[ -z "$BUILDDIR" ] && BUILDDIR=build + +# After some inspection, we've noticed that: +# `pwd` is: /root/src +# $SRCDIR is: /root/src +# $DESTDIR is: /root/dest + +# TODO: Can we use misc/make-path.sh here instead? +# The default that will get used automatically. +export GOPATH=~/go/ + +# The golang bins are stored here. +export GOBIN="${GOPATH}bin/" + +# Needed so that golang build tools will get found. +export PATH="${GOBIN}:${PATH}" + +# Define the golang dir for mgmt. +MGMTDIR="${GOPATH}src/github.com/purpleidea/mgmt/" + +# Make the parent dir. +mkdir -p $(dirname $MGMTDIR) + +# Move our code into it so that golang works. +mv "$SRCDIR" "$MGMTDIR" + +# Work from that directory. +cd "$MGMTDIR" + +# Arch mirror fixes. +if [ -e "/etc/arch-release" ]; then + cp /etc/pacman.d/mirrorlist /etc/pacman.d/mirrorlist.backup + # Enable all mirrors. + sed -i 's/^#Server/Server/' /etc/pacman.d/mirrorlist.backup + #rankmirrors -n 6 /etc/pacman.d/mirrorlist.backup > /etc/pacman.d/mirrorlist # SLOW + echo 'Server = http://mirror.rackspace.com/archlinux/$repo/os/$arch' > /etc/pacman.d/mirrorlist + cat /etc/pacman.d/mirrorlist.backup >> /etc/pacman.d/mirrorlist + pacman -Syu --noconfirm pacman-mirrorlist + pacman -Syyu +fi + +# Get all the dependencies for mgmt. +make deps + +# Build it! +make || exit 1 + +# Store the output. +# TODO: is this required? +mv mgmt $BUILDDIR + +# Get the releases directory path. +releases_path=$(make releases_path) # It captures any other output, careful! +if [ "$releases_path" = "" ]; then + echo "Releases path is unknown." + exit 1 +fi +mkdir -p "$releases_path" # Ensure it exists. + +echo "The releases_path is: ${releases_path}." + +# Build the package for the distribution that we're in. +if [ -e "/etc/redhat-release" ]; then + # TODO: differentiate between Fedora and CentOS + make release_rpm + typ='rpm' +elif [ -e "/etc/debian_version" ]; then + # TODO: differentiate between Debian and Ubuntu + make release_deb + typ='deb' +elif [ -e "/etc/arch-release" ]; then + make release_pacman + typ='pacman' +else + echo "OS family is unknown." + exit 1 +fi +echo "The release type is: ${typ}." + +# Store releases. +mv "$releases_path${typ}/" $BUILDDIR # mv releases/$(VERSION)/rpm/ $BUILDDIR + +echo "Done mkosi build!" diff --git a/misc/mkosi/mkosi.default.archlinux b/misc/mkosi/mkosi.default.archlinux new file mode 100644 index 00000000..4e56d9c6 --- /dev/null +++ b/misc/mkosi/mkosi.default.archlinux @@ -0,0 +1,53 @@ +[Distribution] +Distribution=arch +#Mirror=http://mirror.rackspace.com/archlinux/ + +[Output] +Format=raw_btrfs +Bootable=yes + +# This can be a ramdisk to improve performance. +OutputDirectory=mkosi.output + +[Partitions] +# 1G isn't enough for the base distros. +RootSize=5G + +[Packages] +# We need to download sources from the internet. +WithNetwork=true + +# The packages to appear in both the build and the final image. +Packages= + pacman + pacman-contrib + pacman-mirrorlist + make + git + go + tree +# The packages to appear in the build image, but absent from the final image. +#BuildPackages= +# pacman +# pacman-contrib +# pacman-mirrorlist +# make +# git +# go +# tree + +# Store our build artifacts here. This makes them accessible to the outside. +BuildDirectory=mkosi.builddir + +# Cache some of our package manager downloads. +Cache=mkosi.cache + +# Pull the parent git repository into the image. +BuildSources=../.. + +# Use this transfer mode for the git sources. +# XXX: Does auto-detect even work if BuildSources is not the current dir? Bug? +SourceFileTransfer=copy-git-cached + +# Skip the (second) final build phase, since we only need the build artifact. +SkipFinalPhase=true diff --git a/misc/mkosi/mkosi.default.debian-10 b/misc/mkosi/mkosi.default.debian-10 new file mode 100644 index 00000000..14fcdf26 --- /dev/null +++ b/misc/mkosi/mkosi.default.debian-10 @@ -0,0 +1,49 @@ +[Distribution] +Distribution=debian +Release=stable + +[Output] +Format=raw_btrfs +Bootable=yes + +# This can be a ramdisk to improve performance. +OutputDirectory=mkosi.output + +[Partitions] +# 1G isn't enough for the base distros. +RootSize=5G + +[Packages] +# We need to download sources from the internet. +WithNetwork=true + +# The packages to appear in both the build and the final image. +Packages= + apt + make + git + golang + tree +# The packages to appear in the build image, but absent from the final image. +#BuildPackages= +# apt +# make +# git +# golang +# tree + +# Store our build artifacts here. This makes them accessible to the outside. +BuildDirectory=mkosi.builddir + +# Cache some of our package manager downloads. +Cache=mkosi.cache + +# Pull the parent git repository into the image. +BuildSources=../.. + +# Use this transfer mode for the git sources. +# XXX: Does auto-detect even work if BuildSources is not the current dir? Bug? +SourceFileTransfer=copy-git-cached + +# Skip the (second) final build phase, since we only need the build artifact. +SkipFinalPhase=true diff --git a/misc/mkosi/mkosi.default.fedora-29 b/misc/mkosi/mkosi.default.fedora-29 new file mode 100644 index 00000000..18c7b3d9 --- /dev/null +++ b/misc/mkosi/mkosi.default.fedora-29 @@ -0,0 +1,50 @@ +[Distribution] +Distribution=fedora +Release=29 + +[Output] +Format=raw_btrfs +#Format=gpt_btrfs +Bootable=yes + +# This can be a ramdisk to improve performance. +OutputDirectory=mkosi.output + +[Partitions] +# 1G isn't enough for the base distros. +RootSize=5G + +[Packages] +# We need to download sources from the internet. +WithNetwork=true + +# The packages to appear in both the build and the final image. +Packages= + dnf + make + git + golang + tree +# The packages to appear in the build image, but absent from the final image. +#BuildPackages= +# dnf +# make +# git +# golang +# tree + +# Store our build artifacts here. This makes them accessible to the outside. +BuildDirectory=mkosi.builddir + +# Cache some of our package manager downloads. +Cache=mkosi.cache + +# Pull the parent git repository into the image. +BuildSources=../.. + +# Use this transfer mode for the git sources. +# XXX: Does auto-detect even work if BuildSources is not the current dir? Bug? +SourceFileTransfer=copy-git-cached + +# Skip the (second) final build phase, since we only need the build artifact. +SkipFinalPhase=true