test, integration: Add an integration test framework

This adds an initial implementation of an integration test framework for
writing more complicated tests. In particular this also makes some small
additions to the mgmt core so that testing is easier.

integration/basic_test.go

@@ -0,0 +1,130 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package integration
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"sort"
+	"testing"
+)
+
+func TestInstance0(t *testing.T) {
+	code := `
+	$root = getenv("MGMT_TEST_ROOT")
+
+	file "${root}/mgmt-hello-world" {
+		content => "hello world from @purpleidea\n",
+		state => "exists",
+	}
+	`
+	m := Instance{
+		Hostname: "h1", // arbitrary
+		Preserve: true,
+	}
+	if err := m.SimpleDeployLang(code); err != nil {
+		t.Errorf("failed with: %+v", err)
+		return
+	}
+	d := m.Dir()
+	t.Logf("test ran in:\n%s", d)
+	root := path.Join(d, RootDirectory)
+	file := path.Join(root, "mgmt-hello-world")
+	_, err := os.Stat(file)
+	if err != nil {
+		t.Errorf("could not find file: %+v", err)
+		return
+	}
+}
+
+func TestInstance1(t *testing.T) {
+	type test struct { // an individual test
+		name   string
+		code   string // mcl code
+		fail   bool
+		expect map[string]string
+	}
+	values := []test{}
+
+	{
+		code := Code(`
+		$root = getenv("MGMT_TEST_ROOT")
+
+		file "${root}/mgmt-hello-world" {
+			content => "hello world from @purpleidea\n",
+			state => "exists",
+		}
+		`)
+		values = append(values, test{
+			name: "hello world",
+			code: code,
+			fail: false,
+			expect: map[string]string{
+				"mgmt-hello-world": "hello world from @purpleidea\n",
+			},
+		})
+	}
+
+	for index, test := range values { // run all the tests
+		t.Run(fmt.Sprintf("test #%d (%s)", index, test.name), func(t *testing.T) {
+			code, fail, expect := test.code, test.fail, test.expect
+
+			m := Instance{
+				Hostname: "h1",
+				Preserve: true,
+			}
+			err := m.SimpleDeployLang(code)
+			d := m.Dir()
+			if d != "" {
+				t.Logf("test ran in:\n%s", d)
+			}
+
+			if !fail && err != nil {
+				t.Errorf("failed with: %+v", err)
+				return
+			}
+			if fail && err == nil {
+				t.Errorf("passed, expected fail")
+				return
+			}
+
+			if expect == nil { // test done early
+				return
+			}
+
+			files := []string{}
+			for x := range expect {
+				files = append(files, x)
+			}
+			sort.Strings(files) // loop in a deterministic order
+			for _, f := range files {
+				filename := path.Join(d, RootDirectory, f)
+				b, err := ioutil.ReadFile(filename)
+				if err != nil {
+					t.Errorf("could not read file: `%s`", filename)
+					continue
+				}
+				if expect[f] != string(b) {
+					t.Errorf("file: `%s` did not match expected", f)
+				}
+			}
+		})
+	}
+}

integration/instance.go

@@ -0,0 +1,321 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"sync"
+	"syscall"
+
+	"github.com/purpleidea/mgmt/recwatch"
+
+	errwrap "github.com/pkg/errors"
+)
+
+const (
+	// RootDirectory is the directory that is exposed in the per instance
+	// directory which can be used by that instance safely.
+	RootDirectory = "root"
+
+	// PrefixDirectory is the directory that is exposed in the per instance
+	// directory which is used for the mgmt prefix.
+	PrefixDirectory = "prefix"
+
+	// ConvergedStatusFile is the name of the file which is used for the
+	// converged status tracking.
+	ConvergedStatusFile = "csf.txt"
+
+	// longTimeout is a high bound of time we're willing to wait for events.
+	// If we exceed this timeout, then it's likely we are blocked somewhere.
+	longTimeout = 30 // seconds
+
+	// convergedTimeout is the number of seconds we wait for our instance to
+	// remain unchanged to be considered as converged.
+	convergedTimeout = 5 // seconds
+
+	// dirMode is the the mode used when making directories.
+	dirMode = 0755
+
+	// fileMode is the the mode used when making files.
+	fileMode = 0644
+)
+
+// Instance represents a single running mgmt instance. It is a building block
+// that can be used to run standalone tests, or combined to run clustered tests.
+type Instance struct {
+	// Hostname is a unique identifier for this instance.
+	Hostname string
+
+	// Preserve prevents the runtime output from being explicitly deleted.
+	// This is helpful for running analysis or tests on the output.
+	Preserve bool
+
+	// Debug enables more verbosity.
+	Debug bool
+
+	dir                  string
+	tmpPrefixDirectory   string
+	testRootDirectory    string
+	convergedStatusFile  string
+	convergedStatusIndex int
+
+	cmd *exec.Cmd
+
+	clientURL string // set when launched with run
+}
+
+// Init runs some initialization for this instance. It errors if the struct was
+// populated in an invalid way, or if it can't initialize correctly.
+func (obj *Instance) Init() error {
+	if obj.Hostname == "" {
+		return fmt.Errorf("must specify a hostname")
+	}
+
+	// create temporary directory to use during testing
+	var err error
+	obj.dir, err = ioutil.TempDir("", fmt.Sprintf("mgmt-integration-%s-", obj.Hostname))
+	if err != nil {
+		return errwrap.Wrapf(err, "can't create temporary directory")
+	}
+
+	tmpPrefix := path.Join(obj.dir, PrefixDirectory)
+	if err := os.MkdirAll(tmpPrefix, dirMode); err != nil {
+		return errwrap.Wrapf(err, "can't create prefix directory")
+	}
+	obj.tmpPrefixDirectory = tmpPrefix
+
+	testRootDirectory := path.Join(obj.dir, RootDirectory)
+	if err := os.MkdirAll(testRootDirectory, dirMode); err != nil {
+		return errwrap.Wrapf(err, "can't create instance root directory")
+	}
+	obj.testRootDirectory = testRootDirectory
+
+	obj.convergedStatusFile = path.Join(obj.dir, ConvergedStatusFile)
+
+	return nil
+}
+
+// Close cleans up after we're done with this instance.
+func (obj *Instance) Close() error {
+	if !obj.Preserve {
+		if obj.dir == "" || obj.dir == "/" {
+			panic("obj.dir is set to a dangerous path")
+		}
+		if err := os.RemoveAll(obj.dir); err != nil { // dangerous ;)
+			return errwrap.Wrapf(err, "can't remove instance dir")
+		}
+	}
+	obj.Kill() // safety
+	return nil
+}
+
+// Run launches the instance. It returns an error if it was unable to launch.
+func (obj *Instance) Run(seeds []*Instance) error {
+	if obj.cmd != nil {
+		return fmt.Errorf("an instance is already running")
+	}
+
+	if len(seeds) == 0 {
+		obj.clientURL = "http://127.0.0.1:2379"
+	}
+
+	cmdName, err := BinaryPath()
+	if err != nil {
+		return err
+	}
+	cmdArgs := []string{
+		"run", // mode
+		fmt.Sprintf("--hostname=%s", obj.Hostname),
+		fmt.Sprintf("--prefix=%s", obj.tmpPrefixDirectory),
+		fmt.Sprintf("--converged-timeout=%d", convergedTimeout),
+		"--converged-timeout-no-exit",
+		fmt.Sprintf("--converged-status-file=%s", obj.convergedStatusFile),
+	}
+	if len(seeds) > 0 {
+		urls := []string{}
+		for _, instance := range seeds {
+			if instance.cmd == nil {
+				return fmt.Errorf("instance `%s` has not started yet", instance.Hostname)
+			}
+			urls = append(urls, instance.clientURL)
+		}
+		// TODO: we could just pick the first one instead...
+		//s := fmt.Sprintf("--seeds=%s", urls[0])
+		s := fmt.Sprintf("--seeds=%s", strings.Join(urls, ","))
+		cmdArgs = append(cmdArgs, s)
+	}
+	obj.cmd = exec.Command(cmdName, cmdArgs...)
+	obj.cmd.Env = []string{
+		fmt.Sprintf("MGMT_TEST_ROOT=%s", obj.testRootDirectory),
+	}
+
+	if err := obj.cmd.Start(); err != nil {
+		return errwrap.Wrapf(err, "error starting mgmt")
+	}
+
+	return nil
+}
+
+// Kill the process immediately. This is a `kill -9` for if things get stuck.
+func (obj *Instance) Kill() error {
+	if obj.cmd == nil {
+		return nil // already dead
+	}
+
+	// cause a stack dump first if we can
+	_ = obj.cmd.Process.Signal(syscall.SIGQUIT)
+
+	return obj.cmd.Process.Kill()
+}
+
+// Quit sends a friendly shutdown request to the process. You can specify a
+// context if you'd like to exit earlier. If you trigger an early exit with the
+// context, then this will end up running a `kill -9` so it can return.
+func (obj *Instance) Quit(ctx context.Context) error {
+	if obj.cmd == nil {
+		return fmt.Errorf("no process is running")
+	}
+	if err := obj.cmd.Process.Signal(os.Interrupt); err != nil {
+		return errwrap.Wrapf(err, "could not send interrupt signal")
+	}
+
+	var err error
+	wg := &sync.WaitGroup{}
+	done := make(chan error)
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		done <- obj.cmd.Wait()
+		close(done)
+	}()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		select {
+		case err = <-done:
+		case <-ctx.Done():
+			obj.Kill() // should cause the Wait() to exit
+			err = ctx.Err()
+		}
+	}()
+
+	wg.Wait()
+	obj.cmd = nil
+	return err
+}
+
+// Wait until the first converged state we hit. It is not necessary to use the
+// `--converged-timeout` option with mgmt for this to work. It tracks this via
+// the `--converged-status-file` option which can be used to track the varying
+// convergence status.
+func (obj *Instance) Wait(ctx context.Context) error {
+	//if obj.cmd == nil { // TODO: should we include this?
+	//	return fmt.Errorf("no process is running")
+	//}
+
+	recurse := false
+	recWatcher, err := recwatch.NewRecWatcher(obj.convergedStatusFile, recurse)
+	if err != nil {
+		return errwrap.Wrapf(err, "could not watch file")
+	}
+	defer recWatcher.Close()
+	for {
+		select {
+		case event, ok := <-recWatcher.Events():
+			if !ok {
+				return fmt.Errorf("file watcher shut down")
+			}
+			if err := event.Error; err != nil {
+				return errwrap.Wrapf(err, "error event received")
+			}
+
+			contents, err := ioutil.ReadFile(obj.convergedStatusFile)
+			if err != nil {
+				return errwrap.Wrapf(err, "error reading converged status file")
+			}
+			raw := strings.Split(string(contents), "\n")
+			lines := []string{}
+			for _, x := range raw {
+				if x == "" { // drop blank lines!
+					continue
+				}
+				lines = append(lines, x)
+			}
+
+			if c := len(lines); c < obj.convergedStatusIndex {
+				return fmt.Errorf("file is missing lines or was truncated, got: %d", c)
+			}
+
+			var converged bool
+			for i := obj.convergedStatusIndex; i < len(lines); i++ {
+				obj.convergedStatusIndex = i + 1 // new max
+				line := lines[i]
+				if line == "true" { // converged!
+					converged = true
+				}
+			}
+			if converged {
+				return nil
+			}
+
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
+}
+
+// DeployLang deploys some code to the cluster.
+func (obj *Instance) DeployLang(code string) error {
+	//if obj.cmd == nil { // TODO: should we include this?
+	//	return fmt.Errorf("no process is running")
+	//}
+
+	filename := path.Join(obj.dir, "deploy.mcl")
+	data := []byte(code)
+	if err := ioutil.WriteFile(filename, data, fileMode); err != nil {
+		return err
+	}
+
+	cmdName, err := BinaryPath()
+	if err != nil {
+		return err
+	}
+	cmdArgs := []string{
+		"deploy", // mode
+		"--no-git",
+		"--seeds", obj.clientURL,
+		"lang", "--lang", filename,
+	}
+	cmd := exec.Command(cmdName, cmdArgs...)
+	if err := cmd.Run(); err != nil {
+		return errwrap.Wrapf(err, "can't run deploy")
+	}
+	return nil
+}
+
+// Dir returns the dir where the instance can write to. You should only use this
+// after Init has been called, or it won't have been created and determined yet.
+func (obj *Instance) Dir() string {
+	return obj.dir
+}

integration/patterns.go

@@ -0,0 +1,79 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package integration
+
+import (
+	"context"
+	"time"
+
+	errwrap "github.com/pkg/errors"
+)
+
+// SimpleDeployLang is a helper method that takes a struct and runs a sequence
+// of methods on it. This particular helper starts up an instance, deploys some
+// code, and then shuts down. Both after initially starting up, and after
+// deploy, it waits for the instance to converge before running the next step.
+func (obj *Instance) SimpleDeployLang(code string) error {
+	if err := obj.Init(); err != nil {
+		return errwrap.Wrapf(err, "could not init instance")
+	}
+	defer obj.Close() // clean up working directories
+
+	// run the program
+	if err := obj.Run(nil); err != nil {
+		return errwrap.Wrapf(err, "mgmt could not start")
+	}
+	defer obj.Kill() // do a kill -9
+
+	// wait for an internal converge signal as a baseline
+	{
+		ctx, cancel := context.WithTimeout(context.Background(), longTimeout*time.Second)
+		defer cancel()
+		if err := obj.Wait(ctx); err != nil { // wait to get a converged signal
+			return errwrap.Wrapf(err, "mgmt wait failed") // timeout expired
+		}
+	}
+
+	// push a deploy
+	if err := obj.DeployLang(code); err != nil {
+		return errwrap.Wrapf(err, "mgmt could not deploy")
+	}
+
+	// wait for an internal converge signal
+	{
+		ctx, cancel := context.WithTimeout(context.Background(), longTimeout*time.Second)
+		defer cancel()
+		if err := obj.Wait(ctx); err != nil { // wait to get a converged signal
+			return errwrap.Wrapf(err, "mgmt wait failed") // timeout expired
+		}
+	}
+
+	// press ^C
+	{
+		ctx, cancel := context.WithTimeout(context.Background(), longTimeout*time.Second)
+		defer cancel()
+		if err := obj.Quit(ctx); err != nil {
+			if err == context.DeadlineExceeded {
+				return errwrap.Wrapf(err, "mgmt blocked on exit")
+			}
+			return errwrap.Wrapf(err, "mgmt exited with error")
+		}
+	}
+
+	return nil
+}

integration/util.go

@@ -0,0 +1,75 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package integration
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+const (
+	binaryName = "mgmt"
+)
+
+// BinaryPath returns the full path to an mgmt binary. It expects that someone
+// will run `make build` or something equivalent to produce a binary before this
+// function runs.
+func BinaryPath() (string, error) {
+	_, file, _, ok := runtime.Caller(0)
+	if !ok {
+		return "", fmt.Errorf("can't determine binary path")
+	}
+	dir := filepath.Dir(file) // dir that this file is contained in
+	root := filepath.Dir(dir) // we're in the parent dir to that
+
+	return path.Join(root, binaryName), nil
+}
+
+// Code takes a code block as a backtick enclosed `heredoc` and removes any
+// common indentation from each line. This helps inline code as strings to be
+// formatted nicely without unnecessary indentation. It also drops the very
+// first line of code if it has zero length.
+func Code(code string) string {
+	output := []string{}
+	lines := strings.Split(code, "\n")
+	var found bool
+	var strip string // prefix to remove
+	for i, x := range lines {
+		if !found && len(x) > 0 {
+			for j := 0; j < len(x); j++ {
+				if x[j] != '\t' {
+					break
+				}
+				strip += "\t"
+			}
+			// otherwise, there's no indentation
+			found = true
+		}
+		if i == 0 && len(x) == 0 { // drop first line if it's empty
+			continue
+		}
+
+		s := strings.TrimPrefix(x, strip)
+		output = append(output, s)
+	}
+
+	return strings.Join(output, "\n")
+}

integration/util_test.go

@@ -0,0 +1,65 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package integration
+
+import (
+	"os"
+	"testing"
+)
+
+func TestBinaryPath(t *testing.T) {
+	p, err := BinaryPath()
+	if err != nil {
+		t.Errorf("could not determine binary path: %+v", err)
+		return
+	}
+	if p == "" {
+		t.Errorf("binary path was empty")
+		return
+	}
+	fi, err := os.Stat(p)
+	if err != nil {
+		t.Errorf("could not stat binary path: %+v", err)
+		return
+	}
+	// TODO: check file mode is executable
+	_ = fi
+}
+
+func TestCodeIndent(t *testing.T) {
+	c1 := Code(
+		`
+	$root = getenv("MGMT_TEST_ROOT")
+
+	file "${root}/mgmt-hello-world" {
+		content => "hello world from @purpleidea\n",
+		state => "exists",
+	}
+	`)
+	c2 :=
+		`$root = getenv("MGMT_TEST_ROOT")
+
+file "${root}/mgmt-hello-world" {
+	content => "hello world from @purpleidea\n",
+	state => "exists",
+}
+`
+	if c1 != c2 {
+		t.Errorf("code samples differ")
+	}
+}

lib/cli.go

@@ -104,6 +104,8 @@ func run(c *cli.Context) error {
 	obj.Graphviz = c.String("graphviz")
 	obj.GraphvizFilter = c.String("graphviz-filter")
 	obj.ConvergedTimeout = c.Int("converged-timeout")
+	obj.ConvergedTimeoutNoExit = c.Bool("converged-timeout-no-exit")
+	obj.ConvergedStatusFile = c.String("converged-status-file")
 	obj.MaxRuntime = uint(c.Int("max-runtime"))
 
 	obj.Seeds = c.StringSlice("seeds")
@@ -229,9 +231,18 @@ func CLI(program, version string, flags Flags) error {
 		cli.IntFlag{
 			Name:   "converged-timeout, t",
 			Value:  -1,
-			Usage:  "exit after approximately this many seconds in a converged state",
+			Usage:  "after approximately this many seconds without activity, we're considered to be in a converged state",
 			EnvVar: "MGMT_CONVERGED_TIMEOUT",
 		},
+		cli.BoolFlag{
+			Name:  "converged-timeout-no-exit",
+			Usage: "don't exit on converged-timeout",
+		},
+		cli.StringFlag{
+			Name:  "converged-status-file",
+			Value: "",
+			Usage: "file to append the current converged state to, mostly used for testing",
+		},
 		cli.IntFlag{
 			Name:   "max-runtime",
 			Value:  0,

lib/converged.go

@@ -0,0 +1,38 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package lib
+
+import (
+	"fmt"
+	"os"
+)
+
+// appendConvergedStatus appends the converged status to a file.
+func appendConvergedStatus(filename string, status bool) error {
+	// create or append to the file, in write only mode
+	f, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	// TODO: add a timestamp?
+	byt := []byte(fmt.Sprintf("%t\n", status))
+	if _, err := f.Write(byt); err != nil {
+		return err
+	}
+	return f.Close()
+}

lib/main.go

@@ -71,7 +71,9 @@ type Main struct {
 	Sema                   int    // add a semaphore with this lock count to each resource
 	Graphviz               string // output file for graphviz data
 	GraphvizFilter         string // graphviz filter to use
-	ConvergedTimeout int    // exit after approximately this many seconds in a converged state; -1 to disable
+	ConvergedTimeout       int    // approximately this many seconds of inactivity means we're in a converged state; -1 to disable
+	ConvergedTimeoutNoExit bool   // don't exit on converged timeout
+	ConvergedStatusFile    string // file to append converged status to
 	MaxRuntime             uint   // exit after a maximum of approximately this many seconds
 
 	Seeds               []string // default etc client endpoint
@@ -338,19 +340,33 @@ func (obj *Main) Run() error {
 	time.Sleep(1 * time.Second) // XXX: temporary workaround
 
 	convergerStateFn := func(b bool) error {
+		var err error
+		if obj.ConvergedStatusFile != "" {
+			if obj.Flags.Debug {
+				log.Printf("Main: Converged status is: %t", b)
+			}
+			err = appendConvergedStatus(obj.ConvergedStatusFile, b)
+		}
+
 		// exit if we are using the converged timeout and we are the
 		// root node. otherwise, if we are a child node in a remote
 		// execution hierarchy, we should only notify our converged
 		// state and wait for the parent to trigger the exit.
 		if t := obj.ConvergedTimeout; t >= 0 {
-			if b {
+			if b && !obj.ConvergedTimeoutNoExit {
 				log.Printf("Main: Converged for %d seconds, exiting!", t)
 				obj.Exit(nil) // trigger an exit!
 			}
-			return nil
+			return err
 		}
 		// send our individual state into etcd for others to see
-		return etcd.SetHostnameConverged(EmbdEtcd, hostname, b) // TODO: what should happen on error?
+		e := etcd.SetHostnameConverged(EmbdEtcd, hostname, b) // TODO: what should happen on error?
+		if err == nil {
+			return e
+		} else if e != nil {
+			err = multierr.Append(err, e) // list of errors
+		}
+		return err
 	}
 	if EmbdEtcd != nil {
 		converger.SetStateFn(convergerStateFn)

test.sh

@@ -18,7 +18,7 @@ test -z "$testsuite" && (echo "ENV:"; env; echo; )
 function run-testsuite()
 {
 	testname="$(basename "$1" .sh)"
-	# if not running all test or this test is not explicitly selected, skip it
+	# if not running all tests or if this test is not explicitly selected, skip it
 	if test -z "$testsuite" || test "test-$testsuite" = "$testname";then
 		$@ || failures=$( [ -n "$failures" ] && echo "$failures\\n$@" || echo "$@" )
 	fi
@@ -61,9 +61,13 @@ run-testsuite ./test/test-gotest.sh
 if env | grep -q -e '^TRAVIS=true$' -e '^JENKINS_URL=' -e '^BUILD_TAG=jenkins'; then
 	run-testsuite ./test/test-shell.sh
 	skip-testsuite ./test/test-gotest.sh --race	# XXX: temporarily disabled...
+	run-testsuite ./test/test-integration.sh
+	skip-testsuite ./test/test-integration.sh --race	# XXX: temporarily disabled...
 else
 	REASON="CI server only test" skip-testsuite ./test/test-shell.sh
 	REASON="CI server only test" skip-testsuite ./test/test-gotest.sh --race	# XXX: temporarily disabled...
+	REASON="CI server only test" skip-testsuite ./test/test-integration.sh
+	REASON="CI server only test" skip-testsuite ./test/test-integration.sh --race	# XXX: temporarily disabled...
 fi
 
 run-testsuite ./test/test-gometalinter.sh

test/test-gotest.sh

@@ -14,13 +14,22 @@ function run-test()
 }
 
 base=$(go list .)
-for pkg in `go list -e ./... | grep -v "^${base}/vendor/" | grep -v "^${base}/examples/" | grep -v "^${base}/test/" | grep -v "^${base}/old/" | grep -v "^${base}/tmp/"`; do
+if [[ "$@" = *"--integration"* ]]; then
-	echo -e "\ttesting: $pkg"
+	if [[ "$@" = *"--race"* ]]; then
-	run-test go test "$pkg"
+		run-test go test -race "${base}/integration/"
-	if [ "$1" = "--race" ]; then
+	else
-		run-test go test -race "$pkg"
+		run-test go test "${base}/integration/"
 	fi
-done
+else
+	for pkg in `go list -e ./... | grep -v "^${base}/vendor/" | grep -v "^${base}/examples/" | grep -v "^${base}/test/" | grep -v "^${base}/old/" | grep -v "^${base}/tmp/" | grep -v "^${base}/integration"`; do
+		echo -e "\ttesting: $pkg"
+		if [[ "$@" = *"--race"* ]]; then
+			run-test go test -race "$pkg"
+		else
+			run-test go test "$pkg"
+		fi
+	done
+fi
 
 if [[ -n "$failures" ]]; then
 	echo 'FAIL'

test/test-integration.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# this file exists to that bash completion for test names works
+
+echo running "$0" "$@"
+
+#ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"	# dir!
+ROOT=$(dirname "${BASH_SOURCE}")/..
+cd "${ROOT}"
+. test/util.sh
+
+# this test is handled as a special `go test` test
+exec test/test-gotest.sh --integration $@