Add initial "autoedge" plumbing

This allows for resources to automatically add necessary edges to the
graph so that the event system doesn't have to work overtime due to
sub-optimal execution order.

config.go

@@ -19,6 +19,7 @@ package main
 
 import (
 	"errors"
+	"fmt"
 	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"log"
@@ -115,6 +116,7 @@ func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO
 
 	for _, t := range config.Resources.Noop {
 		obj := NewNoopRes(t.Name)
+		obj.Meta = t.Meta
 		v := g.GetVertexMatch(obj)
 		if v == nil { // no match found
 			v = NewVertex(obj)
@@ -126,6 +128,7 @@ func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO
 
 	for _, t := range config.Resources.Pkg {
 		obj := NewPkgRes(t.Name, t.State, false, false, false)
+		obj.Meta = t.Meta
 		v := g.GetVertexMatch(obj)
 		if v == nil { // no match found
 			v = NewVertex(obj)
@@ -147,6 +150,10 @@ func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO
 			}
 		} else {
 			obj := NewFileRes(t.Name, t.Path, t.Dirname, t.Basename, t.Content, t.State)
+			// XXX: we don't have a way of knowing if any of the
+			// metaparams are undefined, and as a result to set the
+			// defaults that we want! I hate the go yaml parser!!!
+			obj.Meta = t.Meta
 			v := g.GetVertexMatch(obj)
 			if v == nil { // no match found
 				v = NewVertex(obj)
@@ -159,6 +166,7 @@ func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO
 
 	for _, t := range config.Resources.Svc {
 		obj := NewSvcRes(t.Name, t.State, t.Startup)
+		obj.Meta = t.Meta
 		v := g.GetVertexMatch(obj)
 		if v == nil { // no match found
 			v = NewVertex(obj)
@@ -170,6 +178,7 @@ func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO
 
 	for _, t := range config.Resources.Exec {
 		obj := NewExecRes(t.Name, t.Cmd, t.Shell, t.Timeout, t.WatchCmd, t.WatchShell, t.IfCmd, t.IfShell, t.PollInt, t.State)
+		obj.Meta = t.Meta
 		v := g.GetVertexMatch(obj)
 		if v == nil { // no match found
 			v = NewVertex(obj)
@@ -239,5 +248,83 @@ func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO
 		}
 		g.AddEdge(lookup[e.From.Res][e.From.Name], lookup[e.To.Res][e.To.Name], NewEdge(e.Name))
 	}
+
+	// add auto edges
+	log.Println("Compile: Adding AutoEdges...")
+	for _, v := range g.GetVertices() { // for each vertexes autoedges
+		if !v.GetMeta().AutoEdge { // is the metaparam true?
+			continue
+		}
+		autoEdgeObj := v.AutoEdges()
+		if autoEdgeObj == nil {
+			log.Printf("%v[%v]: Config: No auto edges were found!", v.Kind(), v.GetName())
+			continue // next vertex
+		}
+
+		for { // while the autoEdgeObj has more uuids to add...
+			uuids := autoEdgeObj.Next() // get some!
+			if uuids == nil {
+				log.Printf("%v[%v]: Config: The auto edge list is empty!", v.Kind(), v.GetName())
+				break // inner loop
+			}
+			if DEBUG {
+				log.Println("Compile: AutoEdge: UUIDS:")
+				for i, u := range uuids {
+					log.Printf("Compile: AutoEdge: UUID%d: %v", i, u)
+				}
+			}
+
+			// match and add edges
+			result := g.AddEdgesByMatchingUUIDS(v, uuids)
+
+			// report back, and find out if we should continue
+			if !autoEdgeObj.Test(result) {
+				break
+			}
+		}
+	}
+
 	return true
 }
+
+// add edges to the vertex in a graph based on if it matches a uuid list
+func (g *Graph) AddEdgesByMatchingUUIDS(v *Vertex, uuids []ResUUID) []bool {
+	// search for edges and see what matches!
+	var result []bool
+
+	// loop through each uuid, and see if it matches any vertex
+	for _, uuid := range uuids {
+		var found = false
+		// uuid is a ResUUID object
+		for _, vv := range g.GetVertices() { // search
+			if v == vv { // skip self
+				continue
+			}
+			if DEBUG {
+				log.Printf("Compile: AutoEdge: Match: %v[%v] with UUID: %v[%v]", vv.Kind(), vv.GetName(), uuid.Kind(), uuid.GetName())
+			}
+			// we must match to an effective UUID for the resource,
+			// that is to say, the name value of a res is a helpful
+			// handle, but it is not necessarily a unique identity!
+			// remember, resources can return multiple UUID's each!
+			if UUIDExistsInUUIDs(uuid, vv.GetUUIDs()) {
+				// add edge from: vv -> v
+				if uuid.Reversed() {
+					txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", vv.Kind(), vv.GetName(), v.Kind(), v.GetName())
+					log.Printf("Compile: Adding %v", txt)
+					g.AddEdge(vv, v, NewEdge(txt))
+				} else { // edges go the "normal" way, eg: pkg resource
+					txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", v.Kind(), v.GetName(), vv.Kind(), vv.GetName())
+					log.Printf("Compile: Adding %v", txt)
+					g.AddEdge(v, vv, NewEdge(txt))
+				}
+				found = true
+				break
+			}
+		}
+
+		result = append(result, found)
+	}
+
+	return result
+}

examples/autoedges1.yaml

@@ -0,0 +1,19 @@
+---
+graph: mygraph
+resources:
+  file:
+  - name: file1
+    meta:
+      autoedge: true
+    path: "/tmp/foo/bar/f1"
+    content: |
+      i am f1
+    state: exists
+  - name: file2
+    meta:
+      autoedge: true
+    path: "/tmp/foo/"
+    content: |
+      i am f2
+    state: exists
+edges: []

exec.go

@@ -313,6 +313,65 @@ func (obj *ExecRes) CheckApply(apply bool) (stateok bool, err error) {
 	return false, nil    // success
 }
 
+type ExecUUID struct {
+	BaseUUID
+	Cmd   string
+	IfCmd string
+	// TODO: add more elements here
+}
+
+// if and only if they are equivalent, return true
+// if they are not equivalent, return false
+func (obj *ExecUUID) IFF(uuid ResUUID) bool {
+	res, ok := uuid.(*ExecUUID)
+	if !ok {
+		return false
+	}
+	if obj.Cmd != res.Cmd {
+		return false
+	}
+	// TODO: add more checks here
+	//if obj.Shell != res.Shell {
+	//	return false
+	//}
+	//if obj.Timeout != res.Timeout {
+	//	return false
+	//}
+	//if obj.WatchCmd != res.WatchCmd {
+	//	return false
+	//}
+	//if obj.WatchShell != res.WatchShell {
+	//	return false
+	//}
+	if obj.IfCmd != res.IfCmd {
+		return false
+	}
+	//if obj.PollInt != res.PollInt {
+	//	return false
+	//}
+	//if obj.State != res.State {
+	//	return false
+	//}
+	return true
+}
+
+func (obj *ExecRes) AutoEdges() AutoEdge {
+	// TODO: parse as many exec params to look for auto edges, for example
+	// the path of the binary in the Cmd variable might be from in a pkg
+	return nil
+}
+
+// include all params to make a unique identification of this object
+func (obj *ExecRes) GetUUIDs() []ResUUID {
+	x := &ExecUUID{
+		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
+		Cmd:      obj.Cmd,
+		IfCmd:    obj.IfCmd,
+		// TODO: add more params here
+	}
+	return []ResUUID{x}
+}
+
 func (obj *ExecRes) Compare(res Res) bool {
 	switch res.(type) {
 	case *ExecRes:

file.go

@@ -373,6 +373,90 @@ func (obj *FileRes) CheckApply(apply bool) (stateok bool, err error) {
 	return false, nil // success
 }
 
+
+type FileUUID struct {
+	BaseUUID
+	path string
+}
+
+// if and only if they are equivalent, return true
+// if they are not equivalent, return false
+func (obj *FileUUID) IFF(uuid ResUUID) bool {
+	res, ok := uuid.(*FileUUID)
+	if !ok {
+		return false
+	}
+	return obj.path == res.path
+}
+
+type FileResAutoEdges struct {
+	data    []ResUUID
+	pointer int
+	found   bool
+}
+
+func (obj *FileResAutoEdges) Next() []ResUUID {
+	if obj.found {
+		log.Fatal("Shouldn't be called anymore!")
+	}
+	if len(obj.data) == 0 { // check length for rare scenarios
+		return nil
+	}
+	value := obj.data[obj.pointer]
+	obj.pointer += 1
+	return []ResUUID{value} // we return one, even though api supports N
+}
+
+// get results of the earlier Next() call, return if we should continue!
+func (obj *FileResAutoEdges) Test(input []bool) bool {
+	// if there aren't any more remaining
+	if len(obj.data) <= obj.pointer {
+		return false
+	}
+	if obj.found { // already found, done!
+		return false
+	}
+	if len(input) != 1 { // in case we get given bad data
+		log.Fatal("Expecting a single value!")
+	}
+	if input[0] { // if a match is found, we're done!
+		obj.found = true // no more to find!
+		return false
+	}
+	return true // keep going
+}
+
+// generate a simple linear sequence of each parent directory from bottom up!
+func (obj *FileRes) AutoEdges() AutoEdge {
+	var data []ResUUID                             // store linear result chain here...
+	values := PathSplitFullReversed(obj.GetPath()) // build it
+	_, values = values[0], values[1:]              // get rid of first value which is me!
+	for _, x := range values {
+		var reversed bool = true // cheat by passing a pointer
+		data = append(data, &FileUUID{
+			BaseUUID: BaseUUID{
+				name:     obj.GetName(),
+				kind:     obj.Kind(),
+				reversed: &reversed,
+			},
+			path: x, // what matters
+		}) // build list
+	}
+	return &FileResAutoEdges{
+		data:    data,
+		pointer: 0,
+		found:   false,
+	}
+}
+
+func (obj *FileRes) GetUUIDs() []ResUUID {
+	x := &FileUUID{
+		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
+		path:     obj.GetPath(),
+	}
+	return []ResUUID{x}
+}
+
 func (obj *FileRes) Compare(res Res) bool {
 	switch res.(type) {
 	case *FileRes:

misc.go

@@ -27,6 +27,16 @@ import (
 	"time"
 )
 
+// reverse a list of strings
+func ReverseStringList(in []string) []string {
+	var out []string // empty list
+	l := len(in)
+	for i := range in {
+		out = append(out, in[l-i-1])
+	}
+	return out
+}
+
 // Similar to the GNU dirname command
 func Dirname(p string) string {
 	if p == "/" {
@@ -46,6 +56,9 @@ func Basename(p string) string {
 
 // Split a path into an array of tokens excluding any trailing empty tokens
 func PathSplit(p string) []string {
+	if p == "/" { // TODO: can't this all be expressed nicely in one line?
+		return []string{""}
+	}
 	return strings.Split(path.Clean(p), "/")
 }
 
@@ -83,6 +96,22 @@ func PathIsDir(p string) bool {
 	return p[len(p)-1:] == "/" // a dir has a trailing slash in this context
 }
 
+// return the full list of "dependency" paths for a given path in reverse order
+func PathSplitFullReversed(p string) []string {
+	var result []string
+	split := PathSplit(p)
+	count := len(split)
+	var x string
+	for i := 0; i < count; i++ {
+		x = "/" + path.Join(split[0:i+1]...)
+		if i != 0 && !(i+1 == count && !PathIsDir(p)) {
+			x += "/" // add trailing slash
+		}
+		result = append(result, x)
+	}
+	return ReverseStringList(result)
+}
+
 // encode an object as base 64, serialize and then base64 encode
 func ObjToB64(obj interface{}) (string, bool) {
 	b := bytes.Buffer{}

misc_test.go

@@ -19,6 +19,7 @@ package main
 
 import (
 	"fmt"
+	"reflect"
 	"testing"
 )
 
@@ -53,6 +54,13 @@ func TestMiscT1(t *testing.T) {
 func TestMiscT2(t *testing.T) {
 
 	// TODO: compare the output with the actual list
+	p0 := "/"
+	r0 := []string{""} // TODO: is this correct?
+	if len(PathSplit(p0)) != len(r0) {
+		t.Errorf("Result should be: %q.", r0)
+		t.Errorf("Result should have a length of: %v.", len(r0))
+	}
+
 	p1 := "/foo/bar/baz"
 	r1 := []string{"", "foo", "bar", "baz"}
 	if len(PathSplit(p1)) != len(r1) {
@@ -198,3 +206,22 @@ func TestMiscT7(t *testing.T) {
 		t.Errorf("Strings should match.")
 	}
 }
+
+func TestMiscT8(t *testing.T) {
+
+	r0 := []string{"/"}
+	if fullList0 := PathSplitFullReversed("/"); !reflect.DeepEqual(r0, fullList0) {
+		t.Errorf("PathSplitFullReversed expected: %v; got: %v.", r0, fullList0)
+	}
+
+	r1 := []string{"/foo/bar/baz/file", "/foo/bar/baz/", "/foo/bar/", "/foo/", "/"}
+	if fullList1 := PathSplitFullReversed("/foo/bar/baz/file"); !reflect.DeepEqual(r1, fullList1) {
+		t.Errorf("PathSplitFullReversed expected: %v; got: %v.", r1, fullList1)
+	}
+
+	r2 := []string{"/foo/bar/baz/dir/", "/foo/bar/baz/", "/foo/bar/", "/foo/", "/"}
+	if fullList2 := PathSplitFullReversed("/foo/bar/baz/dir/"); !reflect.DeepEqual(r2, fullList2) {
+		t.Errorf("PathSplitFullReversed expected: %v; got: %v.", r2, fullList2)
+	}
+
+}

packagekit.go

@@ -462,6 +462,71 @@ loop:
 	return nil
 }
 
+// get the list of files that are contained inside a list of packageids
+func (bus *Conn) GetFilesByPackageId(packageIds []string) (files map[string][]string, err error) {
+	ch := make(chan *dbus.Signal, PkBufferSize) // we need to buffer :(
+	interfacePath, err := bus.CreateTransaction()
+	if err != nil {
+		return
+	}
+
+	var signals = []string{"Files", "ErrorCode", "Finished", "Destroy"} // "ItemProgress", "Status" ?
+	bus.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
+
+	obj := bus.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
+	call := obj.Call(FmtTransactionMethod("GetFiles"), 0, packageIds)
+	if call.Err != nil {
+		err = call.Err
+		return
+	}
+	files = make(map[string][]string)
+loop:
+	for {
+		// FIXME: add a timeout option to error in case signals are dropped!
+		select {
+		case signal := <-ch:
+
+			if signal.Path != interfacePath {
+				log.Println("PackageKit: Some wires have been crossed!")
+				continue loop
+			}
+
+			if signal.Name == FmtTransactionMethod("ErrorCode") {
+				err = errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body))
+				return
+
+				// one signal returned per packageId found...
+			} else if signal.Name == FmtTransactionMethod("Files") {
+				if len(signal.Body) != 2 { // bad data
+					continue loop
+				}
+				var ok bool
+				var key string
+				var fileList []string
+				if key, ok = signal.Body[0].(string); !ok {
+					continue loop
+				}
+				if fileList, ok = signal.Body[1].([]string); !ok {
+					continue loop // failed conversion
+				}
+				files[key] = fileList // build up map
+
+				continue loop
+			} else if signal.Name == FmtTransactionMethod("Finished") {
+				// TODO: should we wait for the Destroy signal?
+				break loop
+			} else if signal.Name == FmtTransactionMethod("Destroy") {
+				// should already be broken
+				break loop
+			} else {
+				err = errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body))
+				return
+			}
+		}
+	}
+	return
+}
+
 // does flag exist inside data portion of packageId field?
 func FlagInData(flag, data string) bool {
 	flags := strings.Split(data, ":")

pkg.go

@@ -278,6 +278,10 @@ func (obj *PkgRes) CheckApply(apply bool) (stateok bool, err error) {
 	return false, nil // success
 }
 
+func (obj *PkgRes) AutoEdges() AutoEdge {
+	return nil
+}
+
 func (obj *PkgRes) Compare(res Res) bool {
 	switch res.(type) {
 	case *PkgRes:

resources.go

@@ -42,12 +42,40 @@ const (
 	resConvergedTimeout
 )
 
+// a unique identifier for a resource, namely it's name, and the kind ("type")
+type ResUUID interface {
+	GetName() string
+	Kind() string
+	IFF(ResUUID) bool
+
+	Reversed() bool // true means this resource happens before the generator
+}
+
+type BaseUUID struct {
+	name string
+	kind string
+
+	reversed *bool // piggyback edge information here
+}
+
+type AutoEdge interface {
+	Next() []ResUUID  // call to get list of edges to add
+	Test([]bool) bool // call until false
+}
+
+type MetaParams struct {
+	AutoEdge bool `yaml:"autoedge"` // metaparam, should we generate auto edges? // XXX should default to true
+}
+
 type Res interface {
 	Init()
 	GetName() string     // can't be named "Name()" because of struct field
+	GetUUIDs() []ResUUID // most resources only return one
+	GetMeta() MetaParams
 	Kind() string
 	Watch()
 	CheckApply(bool) (bool, error)
+	AutoEdges() AutoEdge
 	SetVertex(*Vertex)
 	SetConvergedCallback(ctimeout int, converged chan bool)
 	Compare(Res) bool
@@ -67,6 +95,7 @@ type Res interface {
 
 type BaseRes struct {
 	Name           string     `yaml:"name"`
+	Meta           MetaParams `yaml:"meta"` // struct of all the metaparams
 	timestamp      int64      // last updated timestamp ?
 	events         chan Event
 	vertex         *Vertex
@@ -95,6 +124,43 @@ func NewNoopRes(name string) *NoopRes {
 	}
 }
 
+// wraps the IFF method when used with a list of UUID's
+func UUIDExistsInUUIDs(uuid ResUUID, uuids []ResUUID) bool {
+	for _, u := range uuids {
+		if uuid.IFF(u) {
+			return true
+		}
+	}
+	return false
+}
+
+func (obj *BaseUUID) GetName() string {
+	return obj.name
+}
+
+func (obj *BaseUUID) Kind() string {
+	return obj.kind
+}
+
+// if and only if they are equivalent, return true
+// if they are not equivalent, return false
+// most resource will want to override this method, since it does the important
+// work of actually discerning if two resources are identical in function
+func (obj *BaseUUID) IFF(uuid ResUUID) bool {
+	res, ok := uuid.(*BaseUUID)
+	if !ok {
+		return false
+	}
+	return obj.name == res.name
+}
+
+func (obj *BaseUUID) Reversed() bool {
+	if obj.reversed == nil {
+		log.Fatal("Programming error!")
+	}
+	return *obj.reversed
+}
+
 // initialize structures like channels if created without New constructor
 func (obj *BaseRes) Init() {
 	obj.events = make(chan Event)
@@ -109,6 +175,10 @@ func (obj *BaseRes) Kind() string {
 	return "Base"
 }
 
+func (obj *BaseRes) GetMeta() MetaParams {
+	return obj.Meta
+}
+
 func (obj *BaseRes) GetVertex() *Vertex {
 	return obj.vertex
 }
@@ -392,6 +462,23 @@ func (obj *NoopRes) CheckApply(apply bool) (stateok bool, err error) {
 	return true, nil // state is always okay
 }
 
+type NoopUUID struct {
+	BaseUUID
+}
+
+func (obj *NoopRes) AutoEdges() AutoEdge {
+	return nil
+}
+
+// include all params to make a unique identification of this object
+// most resources only return one
+func (obj *NoopRes) GetUUIDs() []ResUUID {
+	x := &NoopUUID{
+		BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
+	}
+	return []ResUUID{x}
+}
+
 func (obj *NoopRes) Compare(res Res) bool {
 	switch res.(type) {
 	// we can only compare NoopRes to others of the same resource

svc.go

@@ -308,6 +308,33 @@ func (obj *SvcRes) CheckApply(apply bool) (stateok bool, err error) {
 	return false, nil // success
 }
 
+type SvcUUID struct {
+	BaseUUID
+}
+
+// if and only if they are equivalent, return true
+// if they are not equivalent, return false
+func (obj *SvcUUID) IFF(uuid ResUUID) bool {
+	res, ok := uuid.(*SvcUUID)
+	if !ok {
+		return false
+	}
+	return obj.name == res.name
+}
+
+func (obj *SvcRes) AutoEdges() AutoEdge {
+	// TODO: add auto edges to the files that provide the service files,
+	// which might come from a pkg resource perhaps!
+	return nil
+}
+
+// include all params to make a unique identification of this object
+func (obj *SvcRes) GetUUIDs() []ResUUID {
+	x := &SvcUUID{BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()}}
+	return []ResUUID{x}
+
+}
+
 func (obj *SvcRes) Compare(res Res) bool {
 	switch res.(type) {
 	case *SvcRes: