Add shell based test harness

This allows you to simulate one or more simultaneously running mgmt
processes. It should be easy to use by following the test cases provided.

test.sh

@@ -19,6 +19,5 @@ go vet && echo PASS
 # do these longer tests only when running on ci
 if env | grep -q -e '^TRAVIS=true$' -e '^JENKINS_URL='; then
 	go test -race
-	echo running go vet
+	./test/test-shell.sh
-	go vet && echo PASS
 fi

test/shell/.gitignore

@@ -0,0 +1 @@
+mgmt

test/shell/etcd.sh

@@ -0,0 +1,15 @@
+# NOTE: boiler plate to run etcd; source with: . etcd.sh; should NOT be +x
+set -o errexit
+set -o nounset
+set -o pipefail
+cleanup ()
+{
+	killall etcd || killall -9 etcd || true	# kill etcd
+	rm -rf /tmp/etcd/
+}
+trap cleanup INT QUIT TERM EXIT ERR
+mkdir -p /tmp/etcd/
+cd /tmp/etcd/ >/dev/null	# shush the cd operation
+etcd &	# start etcd as job # 1
+sleep 1s	# let etcd startup
+cd - >/dev/null

test/shell/t1.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# NOTES:
+#	* this is a simple shell based `mgmt` test case
+#	* it is recommended that you run mgmt wrapped in the timeout command
+#	* it is recommended that you run mgmt with --no-watch
+#	* it is recommended that you run mgmt --converged-timeout=<seconds>
+#	* you can run mgmt with --max-runtime=<seconds> in special scenarios
+#	* you can get a separate etcd going by sourcing etcd.sh: . etcd.sh
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+timeout --kill-after=3s 1s ./mgmt --help # hello world!

test/shell/t2.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+. etcd.sh	# start etcd as job # 1
+
+# run till completion
+timeout --kill-after=15s 10s ./mgmt run --file t2.yaml --converged-timeout=5 --no-watch &
+
+#jobs	# etcd is 1
+wait -n 2	# wait for mgmt to exit
+
+test -e /tmp/mgmt/f1
+test -e /tmp/mgmt/f2
+test -e /tmp/mgmt/f3
+test ! -e /tmp/mgmt/f4

test/shell/t2.yaml

@@ -0,0 +1,41 @@
+---
+graph: mygraph
+types:
+  noop:
+  - name: noop1
+  file:
+  - name: file1
+    path: "/tmp/mgmt/f1"
+    content: |
+      i am f1
+    state: exists
+  - name: file2
+    path: "/tmp/mgmt/f2"
+    content: |
+      i am f2
+    state: exists
+  - name: file3
+    path: "/tmp/mgmt/f3"
+    content: |
+      i am f3
+    state: exists
+  - name: file4
+    path: "/tmp/mgmt/f4"
+    content: |
+      i am f4 and i should not be here
+    state: absent
+edges:
+- name: e1
+  from:
+    type: file
+    name: file1
+  to:
+    type: file
+    name: file2
+- name: e2
+  from:
+    type: file
+    name: file2
+  to:
+    type: file
+    name: file3

test/shell/t3-a.yaml

@@ -0,0 +1,28 @@
+---
+graph: mygraph
+types:
+  file:
+  - name: file1a
+    path: "/tmp/mgmt/mgmtA/f1a"
+    content: |
+      i am f1
+    state: exists
+  - name: file2a
+    path: "/tmp/mgmt/mgmtA/f2a"
+    content: |
+      i am f2
+    state: exists
+  - name: "@@file3a"
+    path: "/tmp/mgmt/mgmtA/f3a"
+    content: |
+      i am f3, exported from host A
+    state: exists
+  - name: "@@file4a"
+    path: "/tmp/mgmt/mgmtA/f4a"
+    content: |
+      i am f4, exported from host A
+    state: exists
+collect:
+- type: file
+  pattern: "/tmp/mgmt/mgmtA/"
+edges: []

test/shell/t3-b.yaml

@@ -0,0 +1,28 @@
+---
+graph: mygraph
+types:
+  file:
+  - name: file1b
+    path: "/tmp/mgmt/mgmtB/f1b"
+    content: |
+      i am f1
+    state: exists
+  - name: file2b
+    path: "/tmp/mgmt/mgmtB/f2b"
+    content: |
+      i am f2
+    state: exists
+  - name: "@@file3b"
+    path: "/tmp/mgmt/mgmtB/f3b"
+    content: |
+      i am f3, exported from host B
+    state: exists
+  - name: "@@file4b"
+    path: "/tmp/mgmt/mgmtB/f4b"
+    content: |
+      i am f4, exported from host B
+    state: exists
+collect:
+- type: file
+  pattern: "/tmp/mgmt/mgmtB/"
+edges: []

test/shell/t3-c.yaml

@@ -0,0 +1,28 @@
+---
+graph: mygraph
+types:
+  file:
+  - name: file1c
+    path: "/tmp/mgmt/mgmtC/f1c"
+    content: |
+      i am f1
+    state: exists
+  - name: file2c
+    path: "/tmp/mgmt/mgmtC/f2c"
+    content: |
+      i am f2
+    state: exists
+  - name: "@@file3c"
+    path: "/tmp/mgmt/mgmtC/f3c"
+    content: |
+      i am f3, exported from host C
+    state: exists
+  - name: "@@file4c"
+    path: "/tmp/mgmt/mgmtC/f4c"
+    content: |
+      i am f4, exported from host C
+    state: exists
+collect:
+- type: file
+  pattern: "/tmp/mgmt/mgmtC/"
+edges: []

test/shell/t3.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+. etcd.sh	# start etcd as job # 1
+
+# setup
+mkdir -p "${MGMT_TMPDIR}"mgmt{A..C}
+
+# run till completion
+timeout --kill-after=15s 10s ./mgmt run --file t3-a.yaml --converged-timeout=5 --no-watch &
+timeout --kill-after=15s 10s ./mgmt run --file t3-b.yaml --converged-timeout=5 --no-watch &
+timeout --kill-after=15s 10s ./mgmt run --file t3-c.yaml --converged-timeout=5 --no-watch &
+
+. wait.sh	# wait for everything except etcd
+
+# A: collected
+test -e "${MGMT_TMPDIR}"mgmtA/f3b
+test -e "${MGMT_TMPDIR}"mgmtA/f3c
+test -e "${MGMT_TMPDIR}"mgmtA/f4b
+test -e "${MGMT_TMPDIR}"mgmtA/f4c
+
+# A: local
+test -e "${MGMT_TMPDIR}"mgmtA/f1a
+test -e "${MGMT_TMPDIR}"mgmtA/f2a
+test -e "${MGMT_TMPDIR}"mgmtA/f3a
+test -e "${MGMT_TMPDIR}"mgmtA/f4a
+
+# A: nope!
+test ! -e "${MGMT_TMPDIR}"mgmtA/f1b
+test ! -e "${MGMT_TMPDIR}"mgmtA/f2b
+test ! -e "${MGMT_TMPDIR}"mgmtA/f1c
+test ! -e "${MGMT_TMPDIR}"mgmtA/f2c
+
+# B: collected
+test -e "${MGMT_TMPDIR}"mgmtB/f3a
+test -e "${MGMT_TMPDIR}"mgmtB/f3c
+test -e "${MGMT_TMPDIR}"mgmtB/f4a
+test -e "${MGMT_TMPDIR}"mgmtB/f4c
+
+# B: local
+test -e "${MGMT_TMPDIR}"mgmtB/f1b
+test -e "${MGMT_TMPDIR}"mgmtB/f2b
+test -e "${MGMT_TMPDIR}"mgmtB/f3b
+test -e "${MGMT_TMPDIR}"mgmtB/f4b
+
+# B: nope!
+test ! -e "${MGMT_TMPDIR}"mgmtB/f1a
+test ! -e "${MGMT_TMPDIR}"mgmtB/f2a
+test ! -e "${MGMT_TMPDIR}"mgmtB/f1c
+test ! -e "${MGMT_TMPDIR}"mgmtB/f2c
+
+# C: collected
+test -e "${MGMT_TMPDIR}"mgmtC/f3a
+test -e "${MGMT_TMPDIR}"mgmtC/f3b
+test -e "${MGMT_TMPDIR}"mgmtC/f4a
+test -e "${MGMT_TMPDIR}"mgmtC/f4b
+
+# C: local
+test -e "${MGMT_TMPDIR}"mgmtC/f1c
+test -e "${MGMT_TMPDIR}"mgmtC/f2c
+test -e "${MGMT_TMPDIR}"mgmtC/f3c
+test -e "${MGMT_TMPDIR}"mgmtC/f4c
+
+# C: nope!
+test ! -e "${MGMT_TMPDIR}"mgmtC/f1a
+test ! -e "${MGMT_TMPDIR}"mgmtC/f2a
+test ! -e "${MGMT_TMPDIR}"mgmtC/f1b
+test ! -e "${MGMT_TMPDIR}"mgmtC/f2b

test/shell/wait.sh

@@ -0,0 +1,6 @@
+# NOTE: boiler plate to wait on mgmt; source with: . wait.sh; should NOT be +x
+for j in `jobs -p`
+do
+	[ $j -eq `pidof etcd` ] && continue	# don't wait for etcd
+	wait $j	# wait for mgmt job $j
+done

test/test-shell.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# simple test harness for testing mgmt
+# NOTE: this will rm -rf /tmp/mgmt/
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+LINE=$(printf '=%.0s' `seq -s ' ' $(tput cols)`)	# a terminal width string
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!
+cd "$DIR" >/dev/null	# work from main mgmt directory
+make build
+MGMT="$DIR/test/shell/mgmt"
+cp -a "$DIR/mgmt" "$MGMT"	# put a copy there
+failures=""
+
+# loop through tests
+for i in $DIR/test/shell/*.sh; do
+	[ -x "$i" ] || continue	# file must be executable
+	# TODO: if ARGV has test names, only execute those!
+	ii=`basename "$i"`	# short name
+	cd $DIR/test/shell/ >/dev/null	# shush the cd operation
+	mkdir -p '/tmp/mgmt/'	# directory for mgmt to put files in
+	#echo "Running: $ii"
+	set +o errexit	# don't kill script on test failure
+	export MGMT_TMPDIR='/tmp/mgmt/'	# we can add to env like this
+	out=$($i 2>&1)	# run and capture stdout & stderr
+	e=$?	# save exit code
+	set -o errexit	# re-enable killing on script failure
+	cd - >/dev/null
+	rm -rf '/tmp/mgmt/'	# clean up after test
+	if [ $e -ne 0 ]; then
+		# store failures...
+		failures=$(
+			# prepend previous failures if any
+			[ -n "${failures}" ] && echo "$failures" && echo "$LINE"
+			echo "Script: $ii"
+			# if we see 124, it might be the exit value of timeout!
+			[ $e -eq 124 ] && echo "Exited: $e (timeout?)" || echo "Exited: $e"
+			if [ "$out" = "" ]; then
+				echo "Output: (empty!)"
+			else
+				echo "Output:"
+				echo "$out"
+			fi
+		)
+	else
+		echo -e "ok\t$ii"	# pass
+	fi
+done
+
+# clean up
+rm -f "$MGMT"
+make clean
+
+# display errors
+if [[ -n "${failures}" ]]; then
+	echo 'FAIL'
+	echo 'The following tests failed:'
+	echo "${failures}"
+	exit 1
+fi
+echo PASS