test, integration: Add an integration test framework

This adds an initial implementation of an integration test framework for
writing more complicated tests. In particular this also makes some small
additions to the mgmt core so that testing is easier.

lib/cli.go

@@ -104,6 +104,8 @@ func run(c *cli.Context) error {
 	obj.Graphviz = c.String("graphviz")
 	obj.GraphvizFilter = c.String("graphviz-filter")
 	obj.ConvergedTimeout = c.Int("converged-timeout")
+	obj.ConvergedTimeoutNoExit = c.Bool("converged-timeout-no-exit")
+	obj.ConvergedStatusFile = c.String("converged-status-file")
 	obj.MaxRuntime = uint(c.Int("max-runtime"))
 
 	obj.Seeds = c.StringSlice("seeds")
@@ -229,9 +231,18 @@ func CLI(program, version string, flags Flags) error {
 		cli.IntFlag{
 			Name:   "converged-timeout, t",
 			Value:  -1,
-			Usage:  "exit after approximately this many seconds in a converged state",
+			Usage:  "after approximately this many seconds without activity, we're considered to be in a converged state",
 			EnvVar: "MGMT_CONVERGED_TIMEOUT",
 		},
+		cli.BoolFlag{
+			Name:  "converged-timeout-no-exit",
+			Usage: "don't exit on converged-timeout",
+		},
+		cli.StringFlag{
+			Name:  "converged-status-file",
+			Value: "",
+			Usage: "file to append the current converged state to, mostly used for testing",
+		},
 		cli.IntFlag{
 			Name:   "max-runtime",
 			Value:  0,

lib/converged.go

@@ -0,0 +1,38 @@
+// Mgmt
+// Copyright (C) 2013-2018+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package lib
+
+import (
+	"fmt"
+	"os"
+)
+
+// appendConvergedStatus appends the converged status to a file.
+func appendConvergedStatus(filename string, status bool) error {
+	// create or append to the file, in write only mode
+	f, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	// TODO: add a timestamp?
+	byt := []byte(fmt.Sprintf("%t\n", status))
+	if _, err := f.Write(byt); err != nil {
+		return err
+	}
+	return f.Close()
+}

lib/main.go

@@ -67,12 +67,14 @@ type Main struct {
 	NoConfigWatch bool // do not update graph due to config changes
 	NoStreamWatch bool // do not update graph due to stream changes
 
-	Noop             bool   // globally force all resources into no-op mode
-	Sema             int    // add a semaphore with this lock count to each resource
-	Graphviz         string // output file for graphviz data
-	GraphvizFilter   string // graphviz filter to use
-	ConvergedTimeout int    // exit after approximately this many seconds in a converged state; -1 to disable
-	MaxRuntime       uint   // exit after a maximum of approximately this many seconds
+	Noop                   bool   // globally force all resources into no-op mode
+	Sema                   int    // add a semaphore with this lock count to each resource
+	Graphviz               string // output file for graphviz data
+	GraphvizFilter         string // graphviz filter to use
+	ConvergedTimeout       int    // approximately this many seconds of inactivity means we're in a converged state; -1 to disable
+	ConvergedTimeoutNoExit bool   // don't exit on converged timeout
+	ConvergedStatusFile    string // file to append converged status to
+	MaxRuntime             uint   // exit after a maximum of approximately this many seconds
 
 	Seeds               []string // default etc client endpoint
 	ClientURLs          []string // list of URLs to listen on for client traffic
@@ -338,19 +340,33 @@ func (obj *Main) Run() error {
 	time.Sleep(1 * time.Second) // XXX: temporary workaround
 
 	convergerStateFn := func(b bool) error {
+		var err error
+		if obj.ConvergedStatusFile != "" {
+			if obj.Flags.Debug {
+				log.Printf("Main: Converged status is: %t", b)
+			}
+			err = appendConvergedStatus(obj.ConvergedStatusFile, b)
+		}
+
 		// exit if we are using the converged timeout and we are the
 		// root node. otherwise, if we are a child node in a remote
 		// execution hierarchy, we should only notify our converged
 		// state and wait for the parent to trigger the exit.
 		if t := obj.ConvergedTimeout; t >= 0 {
-			if b {
+			if b && !obj.ConvergedTimeoutNoExit {
 				log.Printf("Main: Converged for %d seconds, exiting!", t)
 				obj.Exit(nil) // trigger an exit!
 			}
-			return nil
+			return err
 		}
 		// send our individual state into etcd for others to see
-		return etcd.SetHostnameConverged(EmbdEtcd, hostname, b) // TODO: what should happen on error?
+		e := etcd.SetHostnameConverged(EmbdEtcd, hostname, b) // TODO: what should happen on error?
+		if err == nil {
+			return e
+		} else if e != nil {
+			err = multierr.Append(err, e) // list of errors
+		}
+		return err
 	}
 	if EmbdEtcd != nil {
 		converger.SetStateFn(convergerStateFn)