test: Streamline test suite a little

This change aims to streamline the integrationtest suite and reduce friction when running (parts of) test suites.

Changes:
- add `test-testname` to makefile to easily run one suite
- made skipping tests first class citizen in test.sh (all available testsuites and the reasons they are skipped are now better exposed and discovered)
- suppress some output of gotest unless there is an error
- no longer build binary for examples and gotest suites
- removed .SILENT from makefile as it being applied to only some targets makes it feel weird (I just learned about this option btw, feel free to comment on this change)
- move individual tests out of `test.sh` and into `test-misc.sh`
- introduced the concept of testsuites to `test.sh`

test/test-bashfmt.sh

@@ -2,7 +2,7 @@
 # check for any bash files that aren't properly formatted
 # TODO: this is hardly exhaustive
 
-echo running test-bashfmt.sh
+echo running "$0"
 set -o errexit
 set -o nounset
 set -o pipefail

test/test-commit-message.sh

@@ -1,6 +1,14 @@
 #!/bin/bash -e
+# tests if commit message conforms to convention
 
-echo running test-commit-message.sh
+# library of utility functions
+# shellcheck disable=SC1091
+. test/util.sh
+
+echo running "$0"
+
+ROOT=$(dirname "${BASH_SOURCE}")/..
+cd "${ROOT}" || exit 1
 
 travis_regex='^\([a-z0-9]\(\(, \)\|[a-z0-9]\)\+[a-z0-9]: \)\+[A-Z0-9][^:]\+[^:.]$'
 

test/test-crossbuild.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 set -e -o pipefail
 

test/test-examples.sh

@@ -1,20 +1,15 @@
 #!/bin/bash
 # check that our examples still build, even if we don't run them here
 
-echo running test-examples.sh
-
-#ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!
-ROOT=$(dirname "${BASH_SOURCE}")/..
-cd "${ROOT}"
+# shellcheck disable=SC1091
 . test/util.sh
 
-failures=''
-function run-test()
-{
-	$@ || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" )
-}
+echo running "$0"
 
-make build
+ROOT=$(dirname "${BASH_SOURCE}")/..
+cd "${ROOT}"
+
+failures=''
 
 buildout='test-examples.out'
 # make symlink to outside of package
@@ -36,11 +31,11 @@ rm `basename "$linkto"`
 cd ..
 rmdir "$tmpdir"	# cleanup
 
-make clean
 if [[ -n "$failures" ]]; then
 	echo 'FAIL'
 	echo "The following tests (in: ${linkto}) have failed:"
 	echo -e "$failures"
+	echo
 	exit 1
 fi
 echo 'PASS'

test/test-gofmt.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # original version of this script from kubernetes project, under ALv2 license
 
-echo running test-gofmt.sh
+echo running "$0"
 set -o errexit
 set -o nounset
 set -o pipefail

test/test-golint.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # check that go lint passes or doesn't get worse by some threshold
 
-echo running test-golint.sh
+echo running "$0"
 
 ORIGPWD=`pwd`
 #ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!

test/test-gometalinter.sh

@@ -2,7 +2,7 @@
 # check a bunch of linters with the gometalinter
 # TODO: run this from the test-golint.sh file instead to check for deltas
 
-echo running test-gometalinter.sh
+echo running "$0"
 
 #ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!
 ROOT=$(dirname "${BASH_SOURCE}")/..
@@ -62,6 +62,7 @@ if [[ -n "$failures" ]]; then
 	echo 'FAIL'
 	echo 'The following tests have failed:'
 	echo -e "$failures"
+	echo
 	exit 1
 fi
 echo 'PASS'

test/test-gotest.sh

@@ -7,29 +7,31 @@ ROOT=$(dirname "${BASH_SOURCE}")/..
 cd "${ROOT}"
 . test/util.sh
 
+tmpdir="`$mktemp --tmpdir -d tmp.XXX`"	# get a dir outside of the main package
+log="$tmpdir/$(basename $0 .sh).log"
+
 failures=''
 function run-test()
 {
 	$@ || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" )
 }
 
-make build
-
 base=$(go list .)
 for pkg in `go list ./... | grep -v "^${base}/vendor/" | grep -v "^${base}/examples/" | grep -v "^${base}/test/" | grep -v "^${base}/old/" | grep -v "^${base}/tmp/"`; do
-	echo "Testing: $pkg"
+	echo -e "\ttesting: $pkg"
 	# FIXME: can we capture and output the stderr from these tests too?
-	run-test go test "$pkg"
+	run-test go test "$pkg" > "$log"
 	if [ "$1" = "--race" ]; then
-		run-test go test -race "$pkg"
+		run-test go test -race "$pkg" > "$log"
 	fi
 done
 
-make clean
 if [[ -n "$failures" ]]; then
 	echo 'FAIL'
+	cat "$log"
 	echo 'The following `go test` runs have failed:'
 	echo -e "$failures"
+	echo
 	exit 1
 fi
 echo 'PASS'

test/test-govet.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # check that go vet passes
 
-echo running test-govet.sh
+echo running "$0"
 
 #ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!
 ROOT=$(dirname "${BASH_SOURCE}")/..
@@ -59,6 +59,7 @@ if [[ -n "$failures" ]]; then
 	echo 'FAIL'
 	echo 'The following tests have failed:'
 	echo -e "$failures"
+	echo
 	exit 1
 fi
 echo 'PASS'

test/test-headerfmt.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # check that headers are properly formatted
 
-echo running test-headerfmt.sh
+echo running "$0"
 
 #ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!
 ROOT=$(dirname "${BASH_SOURCE}")/..

test/test-misc.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# simple tests that don't deserve their own testfile
+
+# library of utility functions
+# shellcheck disable=SC1091
+. test/util.sh
+
+echo running "$0"
+
+ROOT=$(dirname "${BASH_SOURCE}")/..
+cd "${ROOT}" || exit 1
+
+failures=''
+
+# ensure there is no trailing whitespace or other whitespace errors
+run-test git diff-tree --check $(git hash-object -t tree /dev/null) HEAD
+
+# ensure entries to authors file are sorted
+start=$(($(grep -n '^[[:space:]]*$' AUTHORS | awk -F ':' '{print $1}' | head -1) + 1))
+run-test diff <(tail -n +$start AUTHORS | sort) <(tail -n +$start AUTHORS)
+
+if [[ -n "$failures" ]]; then
+	echo 'FAIL'
+	echo "The following tests have failed:"
+	echo -e "$failures"
+	echo
+	exit 1
+fi
+echo 'PASS'

test/test-omv.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -i
 # simple test harness for testing mgmt via omv
-echo running test-omv.sh
+echo running "$0"
 CWD=`pwd`
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"	# dir!
 cd "$DIR" >/dev/null	# work from test directory

test/test-reproducible.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 # simple test for reproducibility, probably needs major improvements
-echo running test-reproducible.sh
+echo running "$0"
 set -o errexit
 set -o pipefail
 

test/test-shell.sh

@@ -2,7 +2,7 @@
 # simple test harness for testing mgmt
 # NOTE: this will rm -rf /tmp/mgmt/
 
-echo running test-shell.sh
+echo running "$0"
 set -o errexit
 set -o pipefail
 
@@ -21,7 +21,6 @@ fi
 LINE=$(printf '=%.0s' `seq -s ' ' $(tput cols)`)	# a terminal width string
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!
 cd "$DIR" >/dev/null	# work from main mgmt directory
-make build
 MGMT="$DIR/test/shell/mgmt"
 cp -a "$DIR/mgmt" "$MGMT"	# put a copy there
 failures=""
@@ -67,10 +66,6 @@ for i in $DIR/test/shell/*.sh; do
 	fi
 done
 
-# clean up
-rm -f "$MGMT"
-make clean
-
 if [ "$count" = '0' ]; then
 	fail_test 'No tests were run!'
 fi

test/test-yamlfmt.sh

@@ -3,7 +3,7 @@
 
 exit 0	# i give up, we're skipping this entirely, help wanted to fix this
 
-echo running test-yamlfmt.sh
+echo running "$0"
 set -o errexit
 set -o nounset
 set -o pipefail

test/util.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # common settings and functions for test scripts
 
 if [[ $(uname) == "Darwin" ]] ; then
@@ -13,3 +15,8 @@ fail_test()
 	echo "FAIL: $@"
 	exit 1
 }
+
+function run-test()
+{
+	"$@" || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" )
+}