test: Streamline test suite a little

This change aims to streamline the integrationtest suite and reduce friction when running (parts of) test suites.

Changes:
- add `test-testname` to makefile to easily run one suite
- made skipping tests first class citizen in test.sh (all available testsuites and the reasons they are skipped are now better exposed and discovered)
- suppress some output of gotest unless there is an error
- no longer build binary for examples and gotest suites
- removed .SILENT from makefile as it being applied to only some targets makes it feel weird (I just learned about this option btw, feel free to comment on this change)
- move individual tests out of `test.sh` and into `test-misc.sh`
- introduced the concept of testsuites to `test.sh`

Makefile

@@ -157,9 +157,20 @@ clean:
 	# crossbuild artifacts
 	rm -f build/mgmt-*
 
-test: bindata
+test: build
 	./test.sh
 
+# create all test targets for make tab completion (eg: make test-gofmt)
+test_suites=$(shell find test/ -maxdepth 1 -name test-* -exec basename {} .sh \;)
+# allow to run only one test suite at a time
+${test_suites}: test-%: build
+	./test.sh $*
+
+# targets to run individual shell tests (eg: make test-shell-load0)
+test_shell=$(shell find test/shell/ -maxdepth 1 -name "*.sh" -exec basename {} .sh \;)
+$(addprefix test-shell-,${test_shell}): test-shell-%: build
+	./test/test-shell.sh "$*.sh"
+
 gofmt:
 	# TODO: remove gofmt once goimports has a -s option
 	find . -maxdepth 6 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec gofmt -s -w {} \;

README.md

@@ -41,6 +41,7 @@ Please read, enjoy and help improve our documentation!
 | [godoc API reference](https://godoc.org/github.com/purpleidea/mgmt) | for mgmt developers |
 | [prometheus guide](docs/prometheus.md) | for everyone |
 | [puppet guide](docs/puppet-guide.md) | for puppet sysadmins |
+| [development](docs/development.md) | for mgmt developers |
 
 ## Questions:
 Please ask in the [community](#community)!

docs/development.md

@@ -0,0 +1,31 @@
+# Development
+This document contains some additional information and help regarding developing `mgmt`. Useful tools, conventions, etc.
+
+Be sure to read [quick start guide](docs/quick-start-guide.md) first.
+
+## Testing
+This project has both unit tests in the form of golang tests and integration tests using shell scripting.
+
+Native golang tests are preferred over tests written in our shell testing framework. Please see https://golang.org/pkg/testing/ for more information.
+
+To run all tests:
+
+```
+make test
+```
+
+There is a library of quick and small integration tests for the language and YAML related things, check out [`test/shell/`](/test/shell). Adding a test is as easy as copying one of the files in [`test/shell/`](/test/shell) and adapting it.
+
+This test suite won't run by default (unless when on CI server) and needs to be called explictly using:
+
+```
+make test-shell
+```
+
+Or run an individual shell test using:
+
+```
+make test-shell-load0
+```
+
+Tip: you can use TAB completion with `make` to quickly get a list of possible individual tests to run.

misc/make-deps.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # setup a simple go environment
 XPWD=`pwd`
 ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!

test.sh

@@ -1,54 +1,94 @@
 #!/bin/bash -e
-# test suite...
-echo running test.sh
-echo "ENV:"
-env
+# runs all (or selected) test suite(s) in test/ and aggregates results
+# Usage:
+#	./test.sh
+#	./test.sh gofmt
 
-failures=''
-function run-test()
+# library of utility functions
+# shellcheck disable=SC1091
+. test/util.sh
+
+# allow specifying a single testsuite to run
+testsuite="$1"
+
+# print environment when running all testsuites
+test -z "$testsuite" && (echo "ENV:"; env; echo; )
+
+# run a test and record failures
+function run-testsuite()
 {
-	$@ || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" )
+	testname="$(basename "$1" .sh)"
+	# if not running all test or this test is not explicitly selected, skip it
+	if test -z "$testsuite" || test "test-$testsuite" = "$testname";then
+		$@ || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" )
+	fi
 }
 
-# ensure there is no trailing whitespace or other whitespace errors
-run-test git diff-tree --check $(git hash-object -t tree /dev/null) HEAD
+# only run test if it is explicitly selected, otherwise report it is skipped
+function skip-testsuite()
+{
+	testname=$(basename "$1" .sh)
+	# show skip message only when running full suite
+	if test -z "$testsuite";then
+		echo skipping "$@" "($REASON)"
+		echo 'SKIP'
+	else
+		# if a skipped suite is explicity called, run it anyway
+		if test "test-$testsuite" == "$testname";then
+			run-testsuite "$@"
+		fi
+	fi
+}
 
-# ensure entries to authors file are sorted
-start=$(($(grep -n '^[[:space:]]*$' AUTHORS | awk -F ':' '{print $1}' | head -1) + 1))
-run-test diff <(tail -n +$start AUTHORS | sort) <(tail -n +$start AUTHORS)
+# used at the end to tell if everything went fine
+failures=''
 
-run-test ./test/test-gofmt.sh
-run-test ./test/test-yamlfmt.sh
-run-test ./test/test-bashfmt.sh
-run-test ./test/test-headerfmt.sh
-run-test ./test/test-commit-message.sh
-run-test ./test/test-govet.sh
-run-test ./test/test-examples.sh
-run-test ./test/test-gotest.sh
+run-testsuite ./test/test-misc.sh
+run-testsuite ./test/test-gofmt.sh
+run-testsuite ./test/test-yamlfmt.sh
+run-testsuite ./test/test-bashfmt.sh
+run-testsuite ./test/test-headerfmt.sh
+run-testsuite ./test/test-commit-message.sh
+run-testsuite ./test/test-govet.sh
+run-testsuite ./test/test-examples.sh
+run-testsuite ./test/test-gotest.sh
 
 # skipping: https://github.com/purpleidea/mgmt/issues/327
 # run-test ./test/test-crossbuild.sh
 
 # do these longer tests only when running on ci
 if env | grep -q -e '^TRAVIS=true$' -e '^JENKINS_URL=' -e '^BUILD_TAG=jenkins'; then
-	run-test ./test/test-shell.sh
-	#run-test ./test/test-gotest.sh --race	# XXX: temporarily disabled...
+	run-testsuite ./test/test-shell.sh
+	skip-testsuite ./test/test-gotest.sh --race	# XXX: temporarily disabled...
+else
+	REASON="CI server only test" skip-testsuite ./test/test-shell.sh
+	REASON="CI server only test" skip-testsuite ./test/test-gotest.sh --race	# XXX: temporarily disabled...
 fi
 
-run-test ./test/test-gometalinter.sh
+run-testsuite ./test/test-gometalinter.sh
+
 # FIXME: this now fails everywhere :(
-#run-test ./test/test-reproducible.sh
+skip-testsuite ./test/test-reproducible.sh
 
 # run omv tests on jenkins physical hosts only
 if env | grep -q -e '^JENKINS_URL=' -e '^BUILD_TAG=jenkins'; then
-	run-test ./test/test-omv.sh
+	run-testsuite ./test/test-omv.sh
+else
+	REASON="CI server only test" skip-testsuite ./test/test-omv.sh
 fi
-run-test ./test/test-golint.sh	# test last, because this test is somewhat arbitrary
+
+REASON="https://github.com/purpleidea/mgmt/issues/327" skip-testsuite ./test/test-crossbuild.sh
+
+run-testsuite ./test/test-golint.sh	# test last, because this test is somewhat arbitrary
 
 if [[ -n "$failures" ]]; then
 	echo 'FAIL'
 	echo 'The following tests have failed:'
 	echo -e "$failures"
+	echo
+	echo 'You can rerun a single suite like so:'
+	echo
+	echo 'make test-gofmt'
 	exit 1
 fi
 echo 'ALL PASSED'

test/test-bashfmt.sh

@@ -2,7 +2,7 @@
 # check for any bash files that aren't properly formatted
 # TODO: this is hardly exhaustive
 
-echo running test-bashfmt.sh
+echo running "$0"
 set -o errexit
 set -o nounset
 set -o pipefail

test/test-commit-message.sh

@@ -1,6 +1,14 @@
 #!/bin/bash -e
+# tests if commit message conforms to convention
 
-echo running test-commit-message.sh
+# library of utility functions
+# shellcheck disable=SC1091
+. test/util.sh
+
+echo running "$0"
+
+ROOT=$(dirname "${BASH_SOURCE}")/..
+cd "${ROOT}" || exit 1
 
 travis_regex='^\([a-z0-9]\(\(, \)\|[a-z0-9]\)\+[a-z0-9]: \)\+[A-Z0-9][^:]\+[^:.]$'
 

test/test-crossbuild.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 set -e -o pipefail
 

test/test-examples.sh

@@ -1,20 +1,15 @@
 #!/bin/bash
 # check that our examples still build, even if we don't run them here
 
-echo running test-examples.sh
-
-#ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!
-ROOT=$(dirname "${BASH_SOURCE}")/..
-cd "${ROOT}"
+# shellcheck disable=SC1091
 . test/util.sh
 
-failures=''
-function run-test()
-{
-	$@ || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" )
-}
+echo running "$0"
 
-make build
+ROOT=$(dirname "${BASH_SOURCE}")/..
+cd "${ROOT}"
+
+failures=''
 
 buildout='test-examples.out'
 # make symlink to outside of package
@@ -36,11 +31,11 @@ rm `basename "$linkto"`
 cd ..
 rmdir "$tmpdir"	# cleanup
 
-make clean
 if [[ -n "$failures" ]]; then
 	echo 'FAIL'
 	echo "The following tests (in: ${linkto}) have failed:"
 	echo -e "$failures"
+	echo
 	exit 1
 fi
 echo 'PASS'

test/test-gofmt.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # original version of this script from kubernetes project, under ALv2 license
 
-echo running test-gofmt.sh
+echo running "$0"
 set -o errexit
 set -o nounset
 set -o pipefail

test/test-golint.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # check that go lint passes or doesn't get worse by some threshold
 
-echo running test-golint.sh
+echo running "$0"
 
 ORIGPWD=`pwd`
 #ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!

test/test-gometalinter.sh

@@ -2,7 +2,7 @@
 # check a bunch of linters with the gometalinter
 # TODO: run this from the test-golint.sh file instead to check for deltas
 
-echo running test-gometalinter.sh
+echo running "$0"
 
 #ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!
 ROOT=$(dirname "${BASH_SOURCE}")/..
@@ -62,6 +62,7 @@ if [[ -n "$failures" ]]; then
 	echo 'FAIL'
 	echo 'The following tests have failed:'
 	echo -e "$failures"
+	echo
 	exit 1
 fi
 echo 'PASS'

test/test-gotest.sh

@@ -7,29 +7,31 @@ ROOT=$(dirname "${BASH_SOURCE}")/..
 cd "${ROOT}"
 . test/util.sh
 
+tmpdir="`$mktemp --tmpdir -d tmp.XXX`"	# get a dir outside of the main package
+log="$tmpdir/$(basename $0 .sh).log"
+
 failures=''
 function run-test()
 {
 	$@ || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" )
 }
 
-make build
-
 base=$(go list .)
 for pkg in `go list ./... | grep -v "^${base}/vendor/" | grep -v "^${base}/examples/" | grep -v "^${base}/test/" | grep -v "^${base}/old/" | grep -v "^${base}/tmp/"`; do
-	echo "Testing: $pkg"
+	echo -e "\ttesting: $pkg"
 	# FIXME: can we capture and output the stderr from these tests too?
-	run-test go test "$pkg"
+	run-test go test "$pkg" > "$log"
 	if [ "$1" = "--race" ]; then
-		run-test go test -race "$pkg"
+		run-test go test -race "$pkg" > "$log"
 	fi
 done
 
-make clean
 if [[ -n "$failures" ]]; then
 	echo 'FAIL'
+	cat "$log"
 	echo 'The following `go test` runs have failed:'
 	echo -e "$failures"
+	echo
 	exit 1
 fi
 echo 'PASS'

test/test-govet.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # check that go vet passes
 
-echo running test-govet.sh
+echo running "$0"
 
 #ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!
 ROOT=$(dirname "${BASH_SOURCE}")/..
@@ -59,6 +59,7 @@ if [[ -n "$failures" ]]; then
 	echo 'FAIL'
 	echo 'The following tests have failed:'
 	echo -e "$failures"
+	echo
 	exit 1
 fi
 echo 'PASS'

test/test-headerfmt.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # check that headers are properly formatted
 
-echo running test-headerfmt.sh
+echo running "$0"
 
 #ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!
 ROOT=$(dirname "${BASH_SOURCE}")/..

test/test-misc.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# simple tests that don't deserve their own testfile
+
+# library of utility functions
+# shellcheck disable=SC1091
+. test/util.sh
+
+echo running "$0"
+
+ROOT=$(dirname "${BASH_SOURCE}")/..
+cd "${ROOT}" || exit 1
+
+failures=''
+
+# ensure there is no trailing whitespace or other whitespace errors
+run-test git diff-tree --check $(git hash-object -t tree /dev/null) HEAD
+
+# ensure entries to authors file are sorted
+start=$(($(grep -n '^[[:space:]]*$' AUTHORS | awk -F ':' '{print $1}' | head -1) + 1))
+run-test diff <(tail -n +$start AUTHORS | sort) <(tail -n +$start AUTHORS)
+
+if [[ -n "$failures" ]]; then
+	echo 'FAIL'
+	echo "The following tests have failed:"
+	echo -e "$failures"
+	echo
+	exit 1
+fi
+echo 'PASS'

test/test-omv.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -i
 # simple test harness for testing mgmt via omv
-echo running test-omv.sh
+echo running "$0"
 CWD=`pwd`
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"	# dir!
 cd "$DIR" >/dev/null	# work from test directory

test/test-reproducible.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # simple test for reproducibility, probably needs major improvements
-echo running test-reproducible.sh
+echo running "$0"
 set -o errexit
 set -o pipefail
 

test/test-shell.sh

@@ -2,7 +2,7 @@
 # simple test harness for testing mgmt
 # NOTE: this will rm -rf /tmp/mgmt/
 
-echo running test-shell.sh
+echo running "$0"
 set -o errexit
 set -o pipefail
 
@@ -21,7 +21,6 @@ fi
 LINE=$(printf '=%.0s' `seq -s ' ' $(tput cols)`)	# a terminal width string
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!
 cd "$DIR" >/dev/null	# work from main mgmt directory
-make build
 MGMT="$DIR/test/shell/mgmt"
 cp -a "$DIR/mgmt" "$MGMT"	# put a copy there
 failures=""
@@ -67,10 +66,6 @@ for i in $DIR/test/shell/*.sh; do
 	fi
 done
 
-# clean up
-rm -f "$MGMT"
-make clean
-
 if [ "$count" = '0' ]; then
 	fail_test 'No tests were run!'
 fi

test/test-yamlfmt.sh

@@ -3,7 +3,7 @@
 
 exit 0	# i give up, we're skipping this entirely, help wanted to fix this
 
-echo running test-yamlfmt.sh
+echo running "$0"
 set -o errexit
 set -o nounset
 set -o pipefail

test/util.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # common settings and functions for test scripts
 
 if [[ $(uname) == "Darwin" ]] ; then
@@ -13,3 +15,8 @@ fail_test()
 	echo "FAIL: $@"
 	exit 1
 }
+
+function run-test()
+{
+	"$@" || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" )
+}