From ba21554c5f78e79970b30c5e20fa7ec106278822 Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Fri, 9 Feb 2018 13:10:59 +0100 Subject: [PATCH] build, docs: Improve macOS building - New docker command for quickly running tasks in a Linux environment. - Updated docs with macOS specific details. - Fixed some test issues. - Add (fallible) macOS test target for Travis. --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .travis.yml | 6 ++++- bindata/Makefile | 2 +- docker/scripts/exec-development | 18 ++++++++++++++ docs/quick-start-guide.md | 41 ++++++++++++++++++++++++++++++++ misc/make-deps.sh | 3 ++- test/shell/augeas-1.sh | 2 +- test/shell/file-mode.sh | 8 ++++++- test/shell/file-move.sh | 8 ++++++- test/shell/file-owner.sh | 2 +- test/shell/prometheus-3.sh | 8 ++++++- test/shell/prometheus-4.sh | 8 ++++++- test/test-examples.sh | 2 +- test/test-golint.sh | 2 +- test/test-govet.sh | 2 +- test/test-reproducible.sh | 2 +- test/util.sh | 2 ++ 17 files changed, 104 insertions(+), 14 deletions(-) create mode 100755 docker/scripts/exec-development diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d5125011..39bf2484 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,7 +1,7 @@ ## Tips: * please read the style guide before submitting your patch: -[docs/style-guide.md](docs/style-guide.md) +[docs/style-guide.md](../docs/style-guide.md) * commit message titles must be in the form: ```topic: Capitalized message with no trailing period``` diff --git a/.travis.yml b/.travis.yml index ebfa1729..45b3bafb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,7 @@ language: go +os: + - linux + - osx go: - 1.8.x - 1.9.x @@ -7,7 +10,7 @@ go_import_path: github.com/purpleidea/mgmt sudo: true dist: trusty before_install: - - sudo apt update + - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then sudo apt update; fi - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch --unshallow install: 'make deps' @@ -17,6 +20,7 @@ matrix: allow_failures: - go: tip - go: 1.9.x + - os: osx notifications: irc: channels: diff --git a/bindata/Makefile b/bindata/Makefile index fd3e6c7f..6f5c222e 100644 --- a/bindata/Makefile +++ b/bindata/Makefile @@ -35,4 +35,4 @@ bindata.go: ../COPYING clean: # remove generated bindata/*.go - @ROOT=$$(dirname "$${BASH_SOURCE}")/.. && rm *.go + @ROOT=$$(dirname "$${BASH_SOURCE}")/.. && rm -f *.go diff --git a/docker/scripts/exec-development b/docker/scripts/exec-development new file mode 100755 index 00000000..5bc93440 --- /dev/null +++ b/docker/scripts/exec-development @@ -0,0 +1,18 @@ +#!/bin/bash + +# runs command provided as argument inside a development (Linux) Docker container + +# Stop on any error +set -e + +script_directory="$( cd "$( dirname "$0" )" && pwd )" +project_directory=$script_directory/../.. + +# Specify the Docker image name +image_name='purpleidea/mgmt:development' + +# Run container in development mode +docker run --rm --name=mgm_development --user=mgmt \ + -v "$project_directory:/go/src/github.com/purpleidea/mgmt/" \ + -w /go/src/github.com/purpleidea/mgmt/ \ + -it "$image_name" /bin/bash -c "$*" diff --git a/docs/quick-start-guide.md b/docs/quick-start-guide.md index 0ec9f311..0e8c3ece 100644 --- a/docs/quick-start-guide.md +++ b/docs/quick-start-guide.md @@ -14,6 +14,7 @@ Once you're familiar with the general idea, please start hacking... * You need golang version 1.8 or greater installed. ** To install on rpm style systems: `sudo dnf install golang` ** To install on apt style systems: `sudo apt install golang` +** To install on macOS systems install [Homebrew](https://brew.sh) and run: `brew install go` * You can run `go version` to check the golang version. * If your distro is tool old, you may need to [download](https://golang.org/dl/) a newer golang version. @@ -97,3 +98,43 @@ At the moment we have: * [Arch](https://aur.archlinux.org/packages/mgmt/) Please contribute more! We'd especially like to see a Debian package! + +## OSX/macOS/Darwin development +Developing and running `mgmt` on macOS is currently not supported (but not discouraged either). Meaning it might work but in the case it doesn't you would have to provide your own patches to fix problems (the project maintainer and community are glad to assist where needed). + +There are currently some issues that make `mgmt` less suitable to run for provisioning macOS (eg: https://github.com/purpleidea/mgmt/issues/33). But as a client to provision remote servers it should run fine. + +Since the primary supported systems are Linux and these are the environments tested for it is wise to run these suites during macOS development as well. To ease this Docker can be levaraged ((Docker for Mac)[https://docs.docker.com/docker-for-mac/]). + +Before running any of the commands below create the development Docker image: + +``` +docker/scripts/build-development +``` + +This image requires updating every time dependencies (`make-deps.sh`) change. + +Then to run the test suite: + +``` +docker run --rm -ti \ + -v $PWD:/go/src/github.com/purpleidea/mgmt/ \ + -w /go/src/github.com/purpleidea/mgmt/ \ + purpleidea/mgmt:development \ + make test +``` + +For convenience this command is wrapped in `docker/scripts/exec-development`. + +Basically any command can be executed this way. Because the repository source is mounted into the Docker container invocation will be quick and allow rapid testing, example: + +``` +docker/scripts/exec-development test/test-shell.sh load0.sh +``` + +Other examples: + +``` +docker/scripts/exec-development make build +docker/scripts/exec-development ./mgmt run --tmp-prefix --lang examples/lang/load0.mcl +``` diff --git a/misc/make-deps.sh b/misc/make-deps.sh index cb1135c5..934d572a 100755 --- a/misc/make-deps.sh +++ b/misc/make-deps.sh @@ -39,7 +39,8 @@ if [ ! -z "$APT" ]; then fi if [ ! -z "$BREW" ]; then - $BREW install libvirt || true + # coreutils contains gtimeout + $BREW install libvirt augeas coreutils || true fi if [ ! -z "$PACMAN" ]; then diff --git a/test/shell/augeas-1.sh b/test/shell/augeas-1.sh index 23585ebb..73b4282b 100755 --- a/test/shell/augeas-1.sh +++ b/test/shell/augeas-1.sh @@ -10,7 +10,7 @@ mkdir -p "${MGMT_TMPDIR}" > "${MGMT_TMPDIR}"sshd_config # run empty graph, with prometheus support -timeout --kill-after=40s 35s ./mgmt run --tmp-prefix --yaml=augeas-1.yaml & +$timeout --kill-after=40s 35s ./mgmt run --tmp-prefix --yaml=augeas-1.yaml & pid=$! sleep 5s # let it converge diff --git a/test/shell/file-mode.sh b/test/shell/file-mode.sh index 9ab16406..0f95a894 100755 --- a/test/shell/file-mode.sh +++ b/test/shell/file-mode.sh @@ -1,9 +1,15 @@ #!/bin/bash -e +if [[ $(uname) == "Darwin" ]] ; then + # https://github.com/purpleidea/mgmt/issues/33 + echo "This test is broken on macOS, skipping!" + exit +fi + set -x # run till completion -timeout --kill-after=40s 35s ./mgmt run --yaml file-mode.yaml --converged-timeout=5 --no-watch --tmp-prefix & +$timeout --kill-after=40s 35s ./mgmt run --yaml file-mode.yaml --converged-timeout=5 --no-watch --tmp-prefix & pid=$! wait $pid # get exit status e=$? diff --git a/test/shell/file-move.sh b/test/shell/file-move.sh index 8f8990bd..c37e2586 100755 --- a/test/shell/file-move.sh +++ b/test/shell/file-move.sh @@ -1,10 +1,16 @@ #!/bin/bash -e +if [[ $(uname) == "Darwin" ]] ; then + # https://github.com/purpleidea/mgmt/issues/33 + echo "This test is broken on macOS, skipping!" + exit +fi + mkdir -p /tmp/mgmt/ rm /tmp/mgmt/f1 || true # run empty graph, with prometheus support -timeout --kill-after=40s 35s ./mgmt run --tmp-prefix --yaml=file-move.yaml 2>&1 | tee /tmp/mgmt/file-move.log & +$timeout --kill-after=40s 35s ./mgmt run --tmp-prefix --yaml=file-move.yaml 2>&1 | tee /tmp/mgmt/file-move.log & pid=$! sleep 5s # let it converge diff --git a/test/shell/file-owner.sh b/test/shell/file-owner.sh index 6c671a66..e7961ce9 100755 --- a/test/shell/file-owner.sh +++ b/test/shell/file-owner.sh @@ -9,7 +9,7 @@ if ! timeout 1s sudo -A true; then fi # run till completion -timeout --kill-after=30s 25s sudo -A ./mgmt run --yaml file-owner.yaml --converged-timeout=5 --no-watch --tmp-prefix & +$timeout --kill-after=30s 25s sudo -A ./mgmt run --yaml file-owner.yaml --converged-timeout=5 --no-watch --tmp-prefix & pid=$! wait $pid # get exit status e=$? diff --git a/test/shell/prometheus-3.sh b/test/shell/prometheus-3.sh index eee8b085..99ec8890 100755 --- a/test/shell/prometheus-3.sh +++ b/test/shell/prometheus-3.sh @@ -1,7 +1,13 @@ #!/bin/bash -e +if [[ $(uname) == "Darwin" ]] ; then + # https://github.com/purpleidea/mgmt/issues/33 + echo "This test is broken on macOS, skipping!" + exit +fi + # run a graph, with prometheus support -timeout --kill-after=40s 35s ./mgmt run --tmp-prefix --no-pgp --prometheus --yaml prometheus-3.yaml & +$timeout --kill-after=40s 35s ./mgmt run --tmp-prefix --no-pgp --prometheus --yaml prometheus-3.yaml & pid=$! sleep 10s # let it converge diff --git a/test/shell/prometheus-4.sh b/test/shell/prometheus-4.sh index 43f6df43..a3b951a0 100755 --- a/test/shell/prometheus-4.sh +++ b/test/shell/prometheus-4.sh @@ -1,7 +1,13 @@ #!/bin/bash -xe +if [[ $(uname) == "Darwin" ]] ; then + # https://github.com/purpleidea/mgmt/issues/33 + echo "This test is broken on macOS, skipping!" + exit +fi + # run a graph, with prometheus support -timeout --kill-after=30s 25s ./mgmt run --tmp-prefix --no-pgp --prometheus --yaml prometheus-4.yaml & +$timeout --kill-after=30s 25s ./mgmt run --tmp-prefix --no-pgp --prometheus --yaml prometheus-4.yaml & pid=$! sleep 10s # let it converge diff --git a/test/test-examples.sh b/test/test-examples.sh index ad05b56a..f81c32b0 100755 --- a/test/test-examples.sh +++ b/test/test-examples.sh @@ -19,7 +19,7 @@ make build buildout='test-examples.out' # make symlink to outside of package linkto="`pwd`/examples/lib/" -tmpdir="`mktemp --tmpdir -d tmp.XXX`" # get a dir outside of the main package +tmpdir="`$mktemp --tmpdir -d tmp.XXX`" # get a dir outside of the main package cd "$tmpdir" ln -s "$linkto" # symlink outside of dir cd `basename "$linkto"` diff --git a/test/test-golint.sh b/test/test-golint.sh index 36db5dca..8c835a62 100755 --- a/test/test-golint.sh +++ b/test/test-golint.sh @@ -31,7 +31,7 @@ COUNT=`echo -e "$LINT" | wc -l` # number of golint problems in current branch [ "$LINT" = "" ] && echo PASS && exit # everything is "perfect" echo "$LINT" # display the issues -T=`mktemp --tmpdir -d tmp.X'X'X` # add quotes to avoid matching three X's +T=`$mktemp --tmpdir -d tmp.X'X'X` # add quotes to avoid matching three X's [ "$T" = "" ] && fail_test "Could not create tmpdir" cd $T || fail_test "Could not change into tmpdir $T" git clone --recursive "${ROOT}" 2>/dev/null # make a copy diff --git a/test/test-govet.sh b/test/test-govet.sh index ae31da2c..d39ab9e0 100755 --- a/test/test-govet.sh +++ b/test/test-govet.sh @@ -25,7 +25,7 @@ function simplify-gocase() { function token-coloncheck() { # add quotes to avoid matching three X's - if grep -Ei "[\/]+[\/]+[ ]*+(FIXME[^:]|TODO[^:]|X"'X'"X[^:])" "$1"; then + if grep -Ei "[\/]+[\/]+[ ]*(FIXME[^:]|TODO[^:]|X"'X'"X[^:])" "$1"; then return 1 # tokens must end with a colon fi return 0 diff --git a/test/test-reproducible.sh b/test/test-reproducible.sh index 6c03cc0f..2135b30f 100755 --- a/test/test-reproducible.sh +++ b/test/test-reproducible.sh @@ -7,7 +7,7 @@ set -o pipefail DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" # dir! cd "$DIR" >/dev/null # work from main mgmt directory make build -T=`mktemp --tmpdir -d tmp.X'X'X` # add quotes to avoid matching three X's +T=`$mktemp --tmpdir -d tmp.X'X'X` # add quotes to avoid matching three X's cp -a ./mgmt "$T"/mgmt.1 make clean make build diff --git a/test/util.sh b/test/util.sh index f6ae85d3..32309abd 100644 --- a/test/util.sh +++ b/test/util.sh @@ -2,8 +2,10 @@ if [[ $(uname) == "Darwin" ]] ; then export timeout="gtimeout" + export mktemp="gmktemp" else export timeout="timeout" + export mktemp="mktemp" fi fail_test()