engine: Resources package rewrite

This giant patch makes some much needed improvements to the code base.

* The engine has been rewritten and lives within engine/graph/
* All of the common interfaces and code now live in engine/
* All of the resources are in one package called engine/resources/
* The Res API can use different "traits" from engine/traits/
* The Res API has been simplified to hide many of the old internals
* The Watch & Process loops were previously inverted, but is now fixed
* The likelihood of package cycles has been reduced drastically
* And much, much more...

Unfortunately, some code had to be temporarily removed. The remote code
had to be taken out, as did the prometheus code. We hope to have these
back in new forms as soon as possible.

test/shell/env0.sh

@@ -9,12 +9,20 @@ set -o pipefail
 regex="123,,:123,321,:true,false:123"
 
 tmpdir="$($mktemp --tmpdir -d tmp.XXX)"
+if [[ ! "$tmpdir" =~ "/tmp" ]]; then
+	echo "unexpected tmpdir in: ${tmpdir}"
+	exit 99
+fi
 
 env TMPDIR="${tmpdir}" TEST=123 EMPTY="" $timeout -sKILL 60s "$MGMT" run --tmp-prefix --converged-timeout=5 --lang env0.mcl
 e=$?
 
 egrep "$regex" "$tmpdir/environ" || fail_test "Could not match '$(cat "$tmpdir/environ")' in '$tmpdir/environ' to '$regex'."
 
+if [ "$tmpdir" = "" ]; then
+	echo "BUG, tried to delete empty string path"
+	exit 99
+fi
 # cleanup if everything went well
 rm -r "$tmpdir"
 

test/shell/exec-fail.sh

@@ -3,7 +3,7 @@
 # should take a few seconds plus converged timeout, and test we don't hang!
 # TODO: should we return a different exit code if the resources fail?
 # TODO: should we be converged if one of the resources has permanently failed?
-$timeout --kill-after=60s 55s "$MGMT" run --yaml exec-fail.yaml --converged-timeout=5 --no-watch --no-pgp --tmp-prefix &
+$timeout --kill-after=120s 75s "$MGMT" run --yaml exec-fail.yaml --converged-timeout=15 --no-watch --no-pgp --tmp-prefix &
 pid=$!
 wait $pid	# get exit status
 exit $?

test/shell/exec-usergroup.sh

@@ -38,7 +38,7 @@ function run_usergroup_test() {
 	setup
 
 	# run till completion
-	sudo -A timeout --kill-after=30s 25s "$MGMT" run --yaml ./exec-usergroup/${graph} --converged-timeout=5 --no-watch --tmp-prefix &
+	sudo -A timeout --kill-after=45s 40s "$MGMT" run --yaml ./exec-usergroup/${graph} --converged-timeout=15 --no-watch --tmp-prefix &
 	pid=$!
 	wait $pid	# get exit status
 	e=$?

test/shell/file-move.sh

@@ -8,30 +8,29 @@ $timeout --kill-after=60s 55s "$MGMT" run --tmp-prefix --yaml=file-move.yaml 2>&
 pid=$!
 sleep 5s	# let it converge
 
-initial=$(grep -c 'file\[file1\]: contentCheckApply(true)' /tmp/mgmt/file-move.log)
+initial=$(grep -c 'file\[file1\]: resource: contentCheckApply(true)' /tmp/mgmt/file-move.log)
 
 mv /tmp/mgmt/f1 /tmp/mgmt/f2
 
-sleep 3
+sleep 3s
 
-after_move_count=$(grep -c 'file\[file1\]: contentCheckApply(true)' /tmp/mgmt/file-move.log)
+after_move_count=$(grep -c 'file\[file1\]: resource: contentCheckApply(true)' /tmp/mgmt/file-move.log)
 
-sleep 3
+sleep 3s
 
 echo f2 > /tmp/mgmt/f2
 
-after_moved_file_count=$(grep -c 'file\[file1\]: contentCheckApply(true)' /tmp/mgmt/file-move.log)
-
+after_moved_file_count=$(grep -c 'file\[file1\]: resource: contentCheckApply(true)' /tmp/mgmt/file-move.log)
 
 if [[ ${after_move_count} -le ${initial} ]]
 then
-	echo File move did not trigger a CheckApply
+	echo 'File move did not trigger a CheckApply'
 	exit 1
 fi
 
 if [[ ${after_moved_file_count} -gt ${after_move_count} ]]
 then
-	echo Changing the moved file did trigger a CheckApply
+	echo 'Changing the moved file did trigger a CheckApply'
 	exit 1
 fi
 

test/shell/graph-exit1.sh

@@ -2,9 +2,11 @@
 
 # should take 15 seconds for longest resources plus startup time to shutdown
 # we don't want the ^C to allow the rest of the graph to continue executing!
-$timeout --kill-after=65s 55s "$MGMT" run --yaml graph-exit1.yaml --no-watch --no-pgp --tmp-prefix &
+# this is a test of mgmt exiting quickly via a fast pause after it sees a ^C
+$timeout --kill-after=60s 55s "$MGMT" run --yaml graph-exit1.yaml --no-watch --no-pgp --tmp-prefix &
 pid=$!
 sleep 5s	# let the initial resources start to run...
 killall -SIGINT mgmt	# send ^C to exit mgmt
+killall -SIGINT mgmt	# send a second ^C to activate fast pause
 wait $pid	# get exit status
 exit $?

test/shell/load0.sh

@@ -9,6 +9,8 @@ fi
 set -o errexit
 set -o pipefail
 
+. ../util.sh
+
 # Expected load average values eg: load average: 1.64306640625, 1.8076171875, 1.82958984375
 # High precision results are preferred (more than the 2 digits in /proc/loadavg at least).
 # Precision varies (eg: 4, 9 or 11 digits). Hence no strict check for precision but
@@ -17,6 +19,10 @@ set -o pipefail
 regex="load average: [0-9]\,[0-9]{3,}, [0-9]\,[0-9]{3,}, [0-9]\,[0-9]{3,}"
 
 tmpdir="$($mktemp --tmpdir -d tmp.XXX)"
+if [[ ! "$tmpdir" =~ "/tmp" ]]; then
+	echo "unexpected tmpdir in: ${tmpdir}"
+	exit 99
+fi
 
 cat > "$tmpdir/load0.mcl" <<EOF
 \$theload = load()
@@ -39,6 +45,10 @@ e=$?
 set +e
 egrep "$regex" "$tmpdir/loadavg" || fail_test "Could not match $tmpdir/loadavg to '$regex'."
 
+if [ "$tmpdir" = "" ]; then
+	echo "BUG, tried to delete empty string path"
+	exit 99
+fi
 # cleanup if everything went well
 rm -r "$tmpdir"
 

test/shell/prometheus-3.sh

@@ -1,5 +1,7 @@
 #!/bin/bash -e
 
+exit 0	# XXX: temporarily disabled until prometheus is added back post refactor
+
 # run a graph, with prometheus support
 $timeout --kill-after=60s 55s "$MGMT" run --tmp-prefix --no-pgp --prometheus --yaml prometheus-3.yaml &
 pid=$!

test/shell/prometheus-4.sh

@@ -1,5 +1,7 @@
 #!/bin/bash -xe
 
+exit 0	# XXX: temporarily disabled until prometheus is added back post refactor
+
 # run a graph, with prometheus support
 $timeout --kill-after=60s 55s "$MGMT" run --tmp-prefix --no-pgp --prometheus --yaml prometheus-4.yaml &
 pid=$!

test/test-crossbuild.sh

@@ -9,6 +9,10 @@ echo running "$(basename "$0")"
 # test if we can build for all OSes and ARCHes.
 
 tmpdir="`$mktemp --tmpdir -d tmp.XXX`"	# get a dir outside of the main package
+if [[ ! "$tmpdir" =~ "/tmp" ]]; then
+	echo "unexpected tmpdir in: ${tmpdir}"
+	exit 99
+fi
 log="$tmpdir/$(basename $0 .sh).log"
 
 set +e
@@ -21,5 +25,10 @@ if [ ! $RET -eq 0 ]; then
 else
 	echo 'PASS'
 fi
+
+if [ "$tmpdir" = "" ]; then
+	echo "BUG, tried to delete empty string path"
+	exit 99
+fi
 rm -rf "$tmpdir"
 exit $RET

test/test-gotest.sh

@@ -16,12 +16,12 @@ function run-test()
 base=$(go list .)
 if [[ "$@" = *"--integration"* ]]; then
 	if [[ "$@" = *"--race"* ]]; then
-		run-test go test -race "${base}/integration/"
+		run-test go test -race "${base}/integration" -v
 	else
-		run-test go test "${base}/integration/"
+		run-test go test "${base}/integration" -v
 	fi
 else
-	for pkg in `go list -e ./... | grep -v "^${base}/vendor/" | grep -v "^${base}/examples/" | grep -v "^${base}/test/" | grep -v "^${base}/old/" | grep -v "^${base}/tmp/" | grep -v "^${base}/integration"`; do
+	for pkg in `go list -e ./... | grep -v "^${base}/vendor/" | grep -v "^${base}/examples/" | grep -v "^${base}/test/" | grep -v "^${base}/old" | grep -v "^${base}/old/" | grep -v "^${base}/tmp" | grep -v "^${base}/tmp/" | grep -v "^${base}/integration"`; do
 		echo -e "\ttesting: $pkg"
 		if [[ "$@" = *"--race"* ]]; then
 			run-test go test -race "$pkg"

test/test-shell.sh

@@ -42,6 +42,9 @@ for i in $DIR/test/shell/*.sh; do
 	e=$?	# save exit code
 	set -o errexit	# re-enable killing on script failure
 	cd - >/dev/null
+	if [ -L '/tmp/mgmt/' ]; then	# this was once a symlink :(
+		fail_test "Can't remove symlink in /tmp/mgmt/"
+	fi
 	rm -rf '/tmp/mgmt/'	# clean up after test
 	if [ $e -ne 0 ]; then
 		echo -e "FAIL\t$ii"	# fail