diff --git a/Makefile b/Makefile index a35b5d58..8de58519 100644 --- a/Makefile +++ b/Makefile @@ -157,9 +157,20 @@ clean: # crossbuild artifacts rm -f build/mgmt-* -test: bindata +test: build ./test.sh +# create all test targets for make tab completion (eg: make test-gofmt) +test_suites=$(shell find test/ -maxdepth 1 -name test-* -exec basename {} .sh \;) +# allow to run only one test suite at a time +${test_suites}: test-%: build + ./test.sh $* + +# targets to run individual shell tests (eg: make test-shell-load0) +test_shell=$(shell find test/shell/ -maxdepth 1 -name "*.sh" -exec basename {} .sh \;) +$(addprefix test-shell-,${test_shell}): test-shell-%: build + ./test/test-shell.sh "$*.sh" + gofmt: # TODO: remove gofmt once goimports has a -s option find . -maxdepth 6 -type f -name '*.go' -not -path './old/*' -not -path './tmp/*' -not -path './vendor/*' -exec gofmt -s -w {} \; diff --git a/README.md b/README.md index e9cd8d28..fcecbd5e 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Please read, enjoy and help improve our documentation! | [godoc API reference](https://godoc.org/github.com/purpleidea/mgmt) | for mgmt developers | | [prometheus guide](docs/prometheus.md) | for everyone | | [puppet guide](docs/puppet-guide.md) | for puppet sysadmins | +| [development](docs/development.md) | for mgmt developers | ## Questions: Please ask in the [community](#community)! diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 00000000..d8c94b28 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,31 @@ +# Development +This document contains some additional information and help regarding developing `mgmt`. Useful tools, conventions, etc. + +Be sure to read [quick start guide](docs/quick-start-guide.md) first. + +## Testing +This project has both unit tests in the form of golang tests and integration tests using shell scripting. + +Native golang tests are preferred over tests written in our shell testing framework. Please see https://golang.org/pkg/testing/ for more information. + +To run all tests: + +``` +make test +``` + +There is a library of quick and small integration tests for the language and YAML related things, check out [`test/shell/`](/test/shell). Adding a test is as easy as copying one of the files in [`test/shell/`](/test/shell) and adapting it. + +This test suite won't run by default (unless when on CI server) and needs to be called explictly using: + +``` +make test-shell +``` + +Or run an individual shell test using: + +``` +make test-shell-load0 +``` + +Tip: you can use TAB completion with `make` to quickly get a list of possible individual tests to run. diff --git a/misc/make-deps.sh b/misc/make-deps.sh index aefac144..8b7b21b9 100755 --- a/misc/make-deps.sh +++ b/misc/make-deps.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # setup a simple go environment XPWD=`pwd` ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" # dir! diff --git a/test.sh b/test.sh index c7711334..c8455a66 100755 --- a/test.sh +++ b/test.sh @@ -1,54 +1,94 @@ #!/bin/bash -e -# test suite... -echo running test.sh -echo "ENV:" -env +# runs all (or selected) test suite(s) in test/ and aggregates results +# Usage: +# ./test.sh +# ./test.sh gofmt -failures='' -function run-test() +# library of utility functions +# shellcheck disable=SC1091 +. test/util.sh + +# allow specifying a single testsuite to run +testsuite="$1" + +# print environment when running all testsuites +test -z "$testsuite" && (echo "ENV:"; env; echo; ) + +# run a test and record failures +function run-testsuite() { - $@ || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" ) + testname="$(basename "$1" .sh)" + # if not running all test or this test is not explicitly selected, skip it + if test -z "$testsuite" || test "test-$testsuite" = "$testname";then + $@ || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" ) + fi } -# ensure there is no trailing whitespace or other whitespace errors -run-test git diff-tree --check $(git hash-object -t tree /dev/null) HEAD +# only run test if it is explicitly selected, otherwise report it is skipped +function skip-testsuite() +{ + testname=$(basename "$1" .sh) + # show skip message only when running full suite + if test -z "$testsuite";then + echo skipping "$@" "($REASON)" + echo 'SKIP' + else + # if a skipped suite is explicity called, run it anyway + if test "test-$testsuite" == "$testname";then + run-testsuite "$@" + fi + fi +} -# ensure entries to authors file are sorted -start=$(($(grep -n '^[[:space:]]*$' AUTHORS | awk -F ':' '{print $1}' | head -1) + 1)) -run-test diff <(tail -n +$start AUTHORS | sort) <(tail -n +$start AUTHORS) +# used at the end to tell if everything went fine +failures='' -run-test ./test/test-gofmt.sh -run-test ./test/test-yamlfmt.sh -run-test ./test/test-bashfmt.sh -run-test ./test/test-headerfmt.sh -run-test ./test/test-commit-message.sh -run-test ./test/test-govet.sh -run-test ./test/test-examples.sh -run-test ./test/test-gotest.sh +run-testsuite ./test/test-misc.sh +run-testsuite ./test/test-gofmt.sh +run-testsuite ./test/test-yamlfmt.sh +run-testsuite ./test/test-bashfmt.sh +run-testsuite ./test/test-headerfmt.sh +run-testsuite ./test/test-commit-message.sh +run-testsuite ./test/test-govet.sh +run-testsuite ./test/test-examples.sh +run-testsuite ./test/test-gotest.sh # skipping: https://github.com/purpleidea/mgmt/issues/327 # run-test ./test/test-crossbuild.sh # do these longer tests only when running on ci if env | grep -q -e '^TRAVIS=true$' -e '^JENKINS_URL=' -e '^BUILD_TAG=jenkins'; then - run-test ./test/test-shell.sh - #run-test ./test/test-gotest.sh --race # XXX: temporarily disabled... + run-testsuite ./test/test-shell.sh + skip-testsuite ./test/test-gotest.sh --race # XXX: temporarily disabled... +else + REASON="CI server only test" skip-testsuite ./test/test-shell.sh + REASON="CI server only test" skip-testsuite ./test/test-gotest.sh --race # XXX: temporarily disabled... fi -run-test ./test/test-gometalinter.sh +run-testsuite ./test/test-gometalinter.sh + # FIXME: this now fails everywhere :( -#run-test ./test/test-reproducible.sh +skip-testsuite ./test/test-reproducible.sh # run omv tests on jenkins physical hosts only if env | grep -q -e '^JENKINS_URL=' -e '^BUILD_TAG=jenkins'; then - run-test ./test/test-omv.sh + run-testsuite ./test/test-omv.sh +else + REASON="CI server only test" skip-testsuite ./test/test-omv.sh fi -run-test ./test/test-golint.sh # test last, because this test is somewhat arbitrary + +REASON="https://github.com/purpleidea/mgmt/issues/327" skip-testsuite ./test/test-crossbuild.sh + +run-testsuite ./test/test-golint.sh # test last, because this test is somewhat arbitrary if [[ -n "$failures" ]]; then echo 'FAIL' echo 'The following tests have failed:' echo -e "$failures" + echo + echo 'You can rerun a single suite like so:' + echo + echo 'make test-gofmt' exit 1 fi echo 'ALL PASSED' diff --git a/test/test-bashfmt.sh b/test/test-bashfmt.sh index c45b95a8..544ff3fc 100755 --- a/test/test-bashfmt.sh +++ b/test/test-bashfmt.sh @@ -2,7 +2,7 @@ # check for any bash files that aren't properly formatted # TODO: this is hardly exhaustive -echo running test-bashfmt.sh +echo running "$0" set -o errexit set -o nounset set -o pipefail diff --git a/test/test-commit-message.sh b/test/test-commit-message.sh index 8269d074..ff21bdec 100755 --- a/test/test-commit-message.sh +++ b/test/test-commit-message.sh @@ -1,6 +1,14 @@ #!/bin/bash -e +# tests if commit message conforms to convention -echo running test-commit-message.sh +# library of utility functions +# shellcheck disable=SC1091 +. test/util.sh + +echo running "$0" + +ROOT=$(dirname "${BASH_SOURCE}")/.. +cd "${ROOT}" || exit 1 travis_regex='^\([a-z0-9]\(\(, \)\|[a-z0-9]\)\+[a-z0-9]: \)\+[A-Z0-9][^:]\+[^:.]$' diff --git a/test/test-crossbuild.sh b/test/test-crossbuild.sh index 38025a58..41b29e58 100755 --- a/test/test-crossbuild.sh +++ b/test/test-crossbuild.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash set -e -o pipefail diff --git a/test/test-examples.sh b/test/test-examples.sh index f81c32b0..07724c05 100755 --- a/test/test-examples.sh +++ b/test/test-examples.sh @@ -1,20 +1,15 @@ #!/bin/bash # check that our examples still build, even if we don't run them here -echo running test-examples.sh - -#ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" # dir! -ROOT=$(dirname "${BASH_SOURCE}")/.. -cd "${ROOT}" +# shellcheck disable=SC1091 . test/util.sh -failures='' -function run-test() -{ - $@ || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" ) -} +echo running "$0" -make build +ROOT=$(dirname "${BASH_SOURCE}")/.. +cd "${ROOT}" + +failures='' buildout='test-examples.out' # make symlink to outside of package @@ -36,11 +31,11 @@ rm `basename "$linkto"` cd .. rmdir "$tmpdir" # cleanup -make clean if [[ -n "$failures" ]]; then echo 'FAIL' echo "The following tests (in: ${linkto}) have failed:" echo -e "$failures" + echo exit 1 fi echo 'PASS' diff --git a/test/test-gofmt.sh b/test/test-gofmt.sh index e2d4030d..150d81c3 100755 --- a/test/test-gofmt.sh +++ b/test/test-gofmt.sh @@ -1,7 +1,7 @@ #!/bin/bash # original version of this script from kubernetes project, under ALv2 license -echo running test-gofmt.sh +echo running "$0" set -o errexit set -o nounset set -o pipefail diff --git a/test/test-golint.sh b/test/test-golint.sh index 8c835a62..7aa6cd43 100755 --- a/test/test-golint.sh +++ b/test/test-golint.sh @@ -1,7 +1,7 @@ #!/bin/bash # check that go lint passes or doesn't get worse by some threshold -echo running test-golint.sh +echo running "$0" ORIGPWD=`pwd` #ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" # dir! diff --git a/test/test-gometalinter.sh b/test/test-gometalinter.sh index 17e4586f..caf061f7 100755 --- a/test/test-gometalinter.sh +++ b/test/test-gometalinter.sh @@ -2,7 +2,7 @@ # check a bunch of linters with the gometalinter # TODO: run this from the test-golint.sh file instead to check for deltas -echo running test-gometalinter.sh +echo running "$0" #ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" # dir! ROOT=$(dirname "${BASH_SOURCE}")/.. @@ -62,6 +62,7 @@ if [[ -n "$failures" ]]; then echo 'FAIL' echo 'The following tests have failed:' echo -e "$failures" + echo exit 1 fi echo 'PASS' diff --git a/test/test-gotest.sh b/test/test-gotest.sh index f4595d6a..e074be36 100755 --- a/test/test-gotest.sh +++ b/test/test-gotest.sh @@ -7,29 +7,31 @@ ROOT=$(dirname "${BASH_SOURCE}")/.. cd "${ROOT}" . test/util.sh +tmpdir="`$mktemp --tmpdir -d tmp.XXX`" # get a dir outside of the main package +log="$tmpdir/$(basename $0 .sh).log" + failures='' function run-test() { $@ || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" ) } -make build - base=$(go list .) for pkg in `go list ./... | grep -v "^${base}/vendor/" | grep -v "^${base}/examples/" | grep -v "^${base}/test/" | grep -v "^${base}/old/" | grep -v "^${base}/tmp/"`; do - echo "Testing: $pkg" + echo -e "\ttesting: $pkg" # FIXME: can we capture and output the stderr from these tests too? - run-test go test "$pkg" + run-test go test "$pkg" > "$log" if [ "$1" = "--race" ]; then - run-test go test -race "$pkg" + run-test go test -race "$pkg" > "$log" fi done -make clean if [[ -n "$failures" ]]; then echo 'FAIL' + cat "$log" echo 'The following `go test` runs have failed:' echo -e "$failures" + echo exit 1 fi echo 'PASS' diff --git a/test/test-govet.sh b/test/test-govet.sh index d39ab9e0..6d8e959f 100755 --- a/test/test-govet.sh +++ b/test/test-govet.sh @@ -1,7 +1,7 @@ #!/bin/bash # check that go vet passes -echo running test-govet.sh +echo running "$0" #ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" # dir! ROOT=$(dirname "${BASH_SOURCE}")/.. @@ -59,6 +59,7 @@ if [[ -n "$failures" ]]; then echo 'FAIL' echo 'The following tests have failed:' echo -e "$failures" + echo exit 1 fi echo 'PASS' diff --git a/test/test-headerfmt.sh b/test/test-headerfmt.sh index a471a107..4abab17b 100755 --- a/test/test-headerfmt.sh +++ b/test/test-headerfmt.sh @@ -1,7 +1,7 @@ #!/bin/bash # check that headers are properly formatted -echo running test-headerfmt.sh +echo running "$0" #ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" # dir! ROOT=$(dirname "${BASH_SOURCE}")/.. diff --git a/test/test-misc.sh b/test/test-misc.sh new file mode 100755 index 00000000..ad06759e --- /dev/null +++ b/test/test-misc.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# simple tests that don't deserve their own testfile + +# library of utility functions +# shellcheck disable=SC1091 +. test/util.sh + +echo running "$0" + +ROOT=$(dirname "${BASH_SOURCE}")/.. +cd "${ROOT}" || exit 1 + +failures='' + +# ensure there is no trailing whitespace or other whitespace errors +run-test git diff-tree --check $(git hash-object -t tree /dev/null) HEAD + +# ensure entries to authors file are sorted +start=$(($(grep -n '^[[:space:]]*$' AUTHORS | awk -F ':' '{print $1}' | head -1) + 1)) +run-test diff <(tail -n +$start AUTHORS | sort) <(tail -n +$start AUTHORS) + +if [[ -n "$failures" ]]; then + echo 'FAIL' + echo "The following tests have failed:" + echo -e "$failures" + echo + exit 1 +fi +echo 'PASS' diff --git a/test/test-omv.sh b/test/test-omv.sh index eda9f442..57e627be 100755 --- a/test/test-omv.sh +++ b/test/test-omv.sh @@ -1,6 +1,6 @@ #!/bin/bash -i # simple test harness for testing mgmt via omv -echo running test-omv.sh +echo running "$0" CWD=`pwd` DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # dir! cd "$DIR" >/dev/null # work from test directory diff --git a/test/test-reproducible.sh b/test/test-reproducible.sh index 2135b30f..3e9c9f4d 100755 --- a/test/test-reproducible.sh +++ b/test/test-reproducible.sh @@ -1,6 +1,6 @@ #!/bin/bash # simple test for reproducibility, probably needs major improvements -echo running test-reproducible.sh +echo running "$0" set -o errexit set -o pipefail diff --git a/test/test-shell.sh b/test/test-shell.sh index 2d5aae3a..d753daee 100755 --- a/test/test-shell.sh +++ b/test/test-shell.sh @@ -2,7 +2,7 @@ # simple test harness for testing mgmt # NOTE: this will rm -rf /tmp/mgmt/ -echo running test-shell.sh +echo running "$0" set -o errexit set -o pipefail @@ -21,7 +21,6 @@ fi LINE=$(printf '=%.0s' `seq -s ' ' $(tput cols)`) # a terminal width string DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" # dir! cd "$DIR" >/dev/null # work from main mgmt directory -make build MGMT="$DIR/test/shell/mgmt" cp -a "$DIR/mgmt" "$MGMT" # put a copy there failures="" @@ -67,10 +66,6 @@ for i in $DIR/test/shell/*.sh; do fi done -# clean up -rm -f "$MGMT" -make clean - if [ "$count" = '0' ]; then fail_test 'No tests were run!' fi diff --git a/test/test-yamlfmt.sh b/test/test-yamlfmt.sh index af838a4a..83923412 100755 --- a/test/test-yamlfmt.sh +++ b/test/test-yamlfmt.sh @@ -3,7 +3,7 @@ exit 0 # i give up, we're skipping this entirely, help wanted to fix this -echo running test-yamlfmt.sh +echo running "$0" set -o errexit set -o nounset set -o pipefail diff --git a/test/util.sh b/test/util.sh old mode 100644 new mode 100755 index 32309abd..66e95128 --- a/test/util.sh +++ b/test/util.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # common settings and functions for test scripts if [[ $(uname) == "Darwin" ]] ; then @@ -13,3 +15,8 @@ fail_test() echo "FAIL: $@" exit 1 } + +function run-test() +{ + "$@" || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" ) +}