From ee56155ec429eda30c43c426b0f10d21bd62601e Mon Sep 17 00:00:00 2001
From: James Shubin <james@shubin.ca>
Date: Thu, 14 Feb 2019 18:24:30 -0500
Subject: [PATCH] test: Split travis tests into three blocks

Our tests were taking near 50 minutes which kills them. This also makes
it easier to spot small issues faster.
---
 .travis.yml | 26 +++++++++++++++----------
 test.sh     | 55 +++++++++++++++++++++++++++++++++--------------------
 2 files changed, 50 insertions(+), 31 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index d140a708..3f670c08 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,10 +1,6 @@
 language: go
 os:
   - linux
-go:
-  - 1.10.x
-  - 1.11.x
-  - tip
 go_import_path: github.com/purpleidea/mgmt
 sudo: true
 dist: xenial
@@ -25,17 +21,27 @@ before_install:
   - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
   - git fetch --unshallow
 install: 'make deps'
-script: 'make test'
 matrix:
   fast_finish: false
   allow_failures:
-    - go: 1.11.x
-    - go: tip
-    - os: osx
+  - go: 1.11.x
+  - go: tip
+  - os: osx
   # include only one build for osx for a quicker build as the nr. of these runners are sparse
   include:
-    - os: osx
-      go: 1.10.x
+  - name: "basic tests"
+    go: 1.10.x
+    env: TEST_BLOCK=basic
+  - name: "shell tests"
+    go: 1.10.x
+    env: TEST_BLOCK=shell
+  - name: "race tests"
+    go: 1.10.x
+    env: TEST_BLOCK=race
+  - go: 1.11.x
+  - go: tip
+  - os: osx
+script: 'TEST_BLOCK="$TEST_BLOCK" make test'
 
 # the "secure" channel value is the result of running: ./misc/travis-encrypt.sh
 # with a value of: irc.freenode.net#mgmtconfig to eliminate noise from forks...
diff --git a/test.sh b/test.sh
index 40c3b5a4..88a986fb 100755
--- a/test.sh
+++ b/test.sh
@@ -14,6 +14,15 @@ testsuite="$1"
 # print environment when running all testsuites
 test -z "$testsuite" && (echo "ENV:"; env; echo; )
 
+# make it easy to split test into blocks
+label-block() {
+	if $(env | grep -q -e '^TEST_BLOCK='"$1"'$') || $(! env | grep -q -e '^TEST_BLOCK=') || $(env | grep -q -e '^TEST_BLOCK=$'); then
+		return 0
+	else
+		return 1	# not my block
+	fi
+}
+
 # run a test and record failures
 function run-testsuite()
 {
@@ -43,27 +52,37 @@ function skip-testsuite()
 # used at the end to tell if everything went fine
 failures=''
 
-run-testsuite ./test/test-vet.sh
-run-testsuite ./test/test-misc.sh
-run-testsuite ./test/test-gofmt.sh
-run-testsuite ./test/test-yamlfmt.sh
-run-testsuite ./test/test-bashfmt.sh
-run-testsuite ./test/test-headerfmt.sh
-run-testsuite ./test/test-markdownlint.sh
-run-testsuite ./test/test-commit-message.sh
-run-testsuite ./test/test-govet.sh
-run-testsuite ./test/test-examples.sh
-run-testsuite ./test/test-gotest.sh
+if label-block "basic"; then
+	run-testsuite ./test/test-vet.sh
+	run-testsuite ./test/test-misc.sh
+	run-testsuite ./test/test-gofmt.sh
+	run-testsuite ./test/test-yamlfmt.sh
+	run-testsuite ./test/test-bashfmt.sh
+	run-testsuite ./test/test-headerfmt.sh
+	run-testsuite ./test/test-markdownlint.sh
+	run-testsuite ./test/test-commit-message.sh
+	run-testsuite ./test/test-govet.sh
+	run-testsuite ./test/test-examples.sh
+	run-testsuite ./test/test-gotest.sh
+	run-testsuite ./test/test-gometalinter.sh
+	run-testsuite ./test/test-golint.sh	# test last, because this test is somewhat arbitrary
+	# FIXME: this now fails everywhere :(
+	skip-testsuite ./test/test-reproducible.sh
+fi
 
 # skipping: https://github.com/purpleidea/mgmt/issues/327
 # run-test ./test/test-crossbuild.sh
 
 # do these longer tests only when running on ci
 if env | grep -q -e '^TRAVIS=true$' -e '^JENKINS_URL=' -e '^BUILD_TAG=jenkins'; then
-	run-testsuite ./test/test-shell.sh
-	run-testsuite ./test/test-gotest.sh --race
-	run-testsuite ./test/test-integration.sh
-	run-testsuite ./test/test-integration.sh --race
+
+	if label-block "shell"; then
+		run-testsuite ./test/test-shell.sh
+	fi
+	if label-block "race"; then
+		run-testsuite ./test/test-gotest.sh --race
+		run-testsuite ./test/test-integration.sh --race
+	fi
 
 	# XXX: fix and enable these on travis (sudo: go: command not found)
 	#run-testsuite ./test/test-gotest.sh --root
@@ -82,10 +101,6 @@ else
 	REASON="CI server only test" skip-testsuite ./test/test-integration.sh --root --race
 fi
 
-run-testsuite ./test/test-gometalinter.sh
-
-# FIXME: this now fails everywhere :(
-skip-testsuite ./test/test-reproducible.sh
 
 # run omv tests on jenkins physical hosts only
 if env | grep -q -e '^JENKINS_URL=' -e '^BUILD_TAG=jenkins'; then
@@ -96,8 +111,6 @@ fi
 
 REASON="https://github.com/purpleidea/mgmt/issues/327" skip-testsuite ./test/test-crossbuild.sh
 
-run-testsuite ./test/test-golint.sh	# test last, because this test is somewhat arbitrary
-
 if [[ -n "$failures" ]]; then
 	echo 'FAIL'
 	echo 'The following tests have failed:'