etcd: Rewrite embed etcd implementation

This is a giant cleanup of the etcd code. The earlier version was
written when I was less experienced with golang.

This is still not perfect, and does contain some races, but at least
it's a decent base to start from. The automatic elastic clustering
should be considered an experimental feature. If you need a more
battle-tested cluster, then you should manage etcd manually and point
mgmt at your existing cluster.

util/util.go

@@ -430,6 +430,21 @@ func TimeAfterOrBlockCtx(ctx context.Context, t int) <-chan struct{} {
 	return ch
 }
 
+// CloseAfter takes a duration, similarly to `time.After`, and returns a channel
+// that closes when either the context is done, or the duration expires.
+func CloseAfter(ctx context.Context, d time.Duration) <-chan struct{} {
+	ch := make(chan struct{})
+	go func() {
+		defer close(ch)
+		select {
+		case <-time.After(d):
+			// done
+		case <-ctx.Done():
+		}
+	}()
+	return ch
+}
+
 // SystemBusPrivateUsable makes using the private bus usable.
 // TODO: should be upstream: https://github.com/godbus/dbus/issues/15
 func SystemBusPrivateUsable() (conn *dbus.Conn, err error) {
@@ -468,6 +483,26 @@ func SessionBusPrivateUsable() (conn *dbus.Conn, err error) {
 	return conn, nil // success
 }
 
+// PriorityStrSliceSort filters any elements matching fn to the end of the list.
+// You can reverse the match result with a not to filter to the front instead!
+// A copy of the list is returned, the original is not modified.
+func PriorityStrSliceSort(input []string, fn func(string) bool) []string {
+	output := []string{}
+	found := []string{}
+	for _, x := range input {
+		if fn(x) { // if we find the key, don't include it just yet
+			found = append(found, x) // save for later
+			continue
+		}
+		output = append(output, x)
+	}
+
+	// include the keys at the end (if found)
+	output = append(output, found...)
+
+	return output
+}
+
 // SortedStrSliceCompare takes two lists of strings and returns whether or not
 // they are equivalent. It will return nil if both sets contain the same
 // elements, regardless of order, and an error if they do not.