From c999f0c2cd40a651c8512b23684c867ca36519ab Mon Sep 17 00:00:00 2001 From: James Shubin Date: Tue, 23 Feb 2016 00:13:58 -0500 Subject: [PATCH] Add initial "autoedge" plumbing This allows for resources to automatically add necessary edges to the graph so that the event system doesn't have to work overtime due to sub-optimal execution order. --- config.go | 87 +++++++++++++++++++++++++++++++++++++ examples/autoedges1.yaml | 19 ++++++++ exec.go | 59 +++++++++++++++++++++++++ file.go | 84 ++++++++++++++++++++++++++++++++++++ misc.go | 29 +++++++++++++ misc_test.go | 27 ++++++++++++ packagekit.go | 65 ++++++++++++++++++++++++++++ pkg.go | 4 ++ resources.go | 93 ++++++++++++++++++++++++++++++++++++++-- svc.go | 27 ++++++++++++ 10 files changed, 491 insertions(+), 3 deletions(-) create mode 100644 examples/autoedges1.yaml diff --git a/config.go b/config.go index 2841b86e..4ae2a9b6 100644 --- a/config.go +++ b/config.go @@ -19,6 +19,7 @@ package main import ( "errors" + "fmt" "gopkg.in/yaml.v2" "io/ioutil" "log" @@ -115,6 +116,7 @@ func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO for _, t := range config.Resources.Noop { obj := NewNoopRes(t.Name) + obj.Meta = t.Meta v := g.GetVertexMatch(obj) if v == nil { // no match found v = NewVertex(obj) @@ -126,6 +128,7 @@ func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO for _, t := range config.Resources.Pkg { obj := NewPkgRes(t.Name, t.State, false, false, false) + obj.Meta = t.Meta v := g.GetVertexMatch(obj) if v == nil { // no match found v = NewVertex(obj) @@ -147,6 +150,10 @@ func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO } } else { obj := NewFileRes(t.Name, t.Path, t.Dirname, t.Basename, t.Content, t.State) + // XXX: we don't have a way of knowing if any of the + // metaparams are undefined, and as a result to set the + // defaults that we want! I hate the go yaml parser!!! + obj.Meta = t.Meta v := g.GetVertexMatch(obj) if v == nil { // no match found v = NewVertex(obj) @@ -159,6 +166,7 @@ func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO for _, t := range config.Resources.Svc { obj := NewSvcRes(t.Name, t.State, t.Startup) + obj.Meta = t.Meta v := g.GetVertexMatch(obj) if v == nil { // no match found v = NewVertex(obj) @@ -170,6 +178,7 @@ func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO for _, t := range config.Resources.Exec { obj := NewExecRes(t.Name, t.Cmd, t.Shell, t.Timeout, t.WatchCmd, t.WatchShell, t.IfCmd, t.IfShell, t.PollInt, t.State) + obj.Meta = t.Meta v := g.GetVertexMatch(obj) if v == nil { // no match found v = NewVertex(obj) @@ -239,5 +248,83 @@ func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO } g.AddEdge(lookup[e.From.Res][e.From.Name], lookup[e.To.Res][e.To.Name], NewEdge(e.Name)) } + + // add auto edges + log.Println("Compile: Adding AutoEdges...") + for _, v := range g.GetVertices() { // for each vertexes autoedges + if !v.GetMeta().AutoEdge { // is the metaparam true? + continue + } + autoEdgeObj := v.AutoEdges() + if autoEdgeObj == nil { + log.Printf("%v[%v]: Config: No auto edges were found!", v.Kind(), v.GetName()) + continue // next vertex + } + + for { // while the autoEdgeObj has more uuids to add... + uuids := autoEdgeObj.Next() // get some! + if uuids == nil { + log.Printf("%v[%v]: Config: The auto edge list is empty!", v.Kind(), v.GetName()) + break // inner loop + } + if DEBUG { + log.Println("Compile: AutoEdge: UUIDS:") + for i, u := range uuids { + log.Printf("Compile: AutoEdge: UUID%d: %v", i, u) + } + } + + // match and add edges + result := g.AddEdgesByMatchingUUIDS(v, uuids) + + // report back, and find out if we should continue + if !autoEdgeObj.Test(result) { + break + } + } + } + return true } + +// add edges to the vertex in a graph based on if it matches a uuid list +func (g *Graph) AddEdgesByMatchingUUIDS(v *Vertex, uuids []ResUUID) []bool { + // search for edges and see what matches! + var result []bool + + // loop through each uuid, and see if it matches any vertex + for _, uuid := range uuids { + var found = false + // uuid is a ResUUID object + for _, vv := range g.GetVertices() { // search + if v == vv { // skip self + continue + } + if DEBUG { + log.Printf("Compile: AutoEdge: Match: %v[%v] with UUID: %v[%v]", vv.Kind(), vv.GetName(), uuid.Kind(), uuid.GetName()) + } + // we must match to an effective UUID for the resource, + // that is to say, the name value of a res is a helpful + // handle, but it is not necessarily a unique identity! + // remember, resources can return multiple UUID's each! + if UUIDExistsInUUIDs(uuid, vv.GetUUIDs()) { + // add edge from: vv -> v + if uuid.Reversed() { + txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", vv.Kind(), vv.GetName(), v.Kind(), v.GetName()) + log.Printf("Compile: Adding %v", txt) + g.AddEdge(vv, v, NewEdge(txt)) + } else { // edges go the "normal" way, eg: pkg resource + txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", v.Kind(), v.GetName(), vv.Kind(), vv.GetName()) + log.Printf("Compile: Adding %v", txt) + g.AddEdge(v, vv, NewEdge(txt)) + } + found = true + break + } + } + + result = append(result, found) + } + + return result +} diff --git a/examples/autoedges1.yaml b/examples/autoedges1.yaml new file mode 100644 index 00000000..e9749ac2 --- /dev/null +++ b/examples/autoedges1.yaml @@ -0,0 +1,19 @@ +--- +graph: mygraph +resources: + file: + - name: file1 + meta: + autoedge: true + path: "/tmp/foo/bar/f1" + content: | + i am f1 + state: exists + - name: file2 + meta: + autoedge: true + path: "/tmp/foo/" + content: | + i am f2 + state: exists +edges: [] diff --git a/exec.go b/exec.go index 75b0f41f..9a355410 100644 --- a/exec.go +++ b/exec.go @@ -313,6 +313,65 @@ func (obj *ExecRes) CheckApply(apply bool) (stateok bool, err error) { return false, nil // success } +type ExecUUID struct { + BaseUUID + Cmd string + IfCmd string + // TODO: add more elements here +} + +// if and only if they are equivalent, return true +// if they are not equivalent, return false +func (obj *ExecUUID) IFF(uuid ResUUID) bool { + res, ok := uuid.(*ExecUUID) + if !ok { + return false + } + if obj.Cmd != res.Cmd { + return false + } + // TODO: add more checks here + //if obj.Shell != res.Shell { + // return false + //} + //if obj.Timeout != res.Timeout { + // return false + //} + //if obj.WatchCmd != res.WatchCmd { + // return false + //} + //if obj.WatchShell != res.WatchShell { + // return false + //} + if obj.IfCmd != res.IfCmd { + return false + } + //if obj.PollInt != res.PollInt { + // return false + //} + //if obj.State != res.State { + // return false + //} + return true +} + +func (obj *ExecRes) AutoEdges() AutoEdge { + // TODO: parse as many exec params to look for auto edges, for example + // the path of the binary in the Cmd variable might be from in a pkg + return nil +} + +// include all params to make a unique identification of this object +func (obj *ExecRes) GetUUIDs() []ResUUID { + x := &ExecUUID{ + BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()}, + Cmd: obj.Cmd, + IfCmd: obj.IfCmd, + // TODO: add more params here + } + return []ResUUID{x} +} + func (obj *ExecRes) Compare(res Res) bool { switch res.(type) { case *ExecRes: diff --git a/file.go b/file.go index d115bb9f..d111e99f 100644 --- a/file.go +++ b/file.go @@ -373,6 +373,90 @@ func (obj *FileRes) CheckApply(apply bool) (stateok bool, err error) { return false, nil // success } + +type FileUUID struct { + BaseUUID + path string +} + +// if and only if they are equivalent, return true +// if they are not equivalent, return false +func (obj *FileUUID) IFF(uuid ResUUID) bool { + res, ok := uuid.(*FileUUID) + if !ok { + return false + } + return obj.path == res.path +} + +type FileResAutoEdges struct { + data []ResUUID + pointer int + found bool +} + +func (obj *FileResAutoEdges) Next() []ResUUID { + if obj.found { + log.Fatal("Shouldn't be called anymore!") + } + if len(obj.data) == 0 { // check length for rare scenarios + return nil + } + value := obj.data[obj.pointer] + obj.pointer += 1 + return []ResUUID{value} // we return one, even though api supports N +} + +// get results of the earlier Next() call, return if we should continue! +func (obj *FileResAutoEdges) Test(input []bool) bool { + // if there aren't any more remaining + if len(obj.data) <= obj.pointer { + return false + } + if obj.found { // already found, done! + return false + } + if len(input) != 1 { // in case we get given bad data + log.Fatal("Expecting a single value!") + } + if input[0] { // if a match is found, we're done! + obj.found = true // no more to find! + return false + } + return true // keep going +} + +// generate a simple linear sequence of each parent directory from bottom up! +func (obj *FileRes) AutoEdges() AutoEdge { + var data []ResUUID // store linear result chain here... + values := PathSplitFullReversed(obj.GetPath()) // build it + _, values = values[0], values[1:] // get rid of first value which is me! + for _, x := range values { + var reversed bool = true // cheat by passing a pointer + data = append(data, &FileUUID{ + BaseUUID: BaseUUID{ + name: obj.GetName(), + kind: obj.Kind(), + reversed: &reversed, + }, + path: x, // what matters + }) // build list + } + return &FileResAutoEdges{ + data: data, + pointer: 0, + found: false, + } +} + +func (obj *FileRes) GetUUIDs() []ResUUID { + x := &FileUUID{ + BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()}, + path: obj.GetPath(), + } + return []ResUUID{x} +} + func (obj *FileRes) Compare(res Res) bool { switch res.(type) { case *FileRes: diff --git a/misc.go b/misc.go index 94a281fb..77b17844 100644 --- a/misc.go +++ b/misc.go @@ -27,6 +27,16 @@ import ( "time" ) +// reverse a list of strings +func ReverseStringList(in []string) []string { + var out []string // empty list + l := len(in) + for i := range in { + out = append(out, in[l-i-1]) + } + return out +} + // Similar to the GNU dirname command func Dirname(p string) string { if p == "/" { @@ -46,6 +56,9 @@ func Basename(p string) string { // Split a path into an array of tokens excluding any trailing empty tokens func PathSplit(p string) []string { + if p == "/" { // TODO: can't this all be expressed nicely in one line? + return []string{""} + } return strings.Split(path.Clean(p), "/") } @@ -83,6 +96,22 @@ func PathIsDir(p string) bool { return p[len(p)-1:] == "/" // a dir has a trailing slash in this context } +// return the full list of "dependency" paths for a given path in reverse order +func PathSplitFullReversed(p string) []string { + var result []string + split := PathSplit(p) + count := len(split) + var x string + for i := 0; i < count; i++ { + x = "/" + path.Join(split[0:i+1]...) + if i != 0 && !(i+1 == count && !PathIsDir(p)) { + x += "/" // add trailing slash + } + result = append(result, x) + } + return ReverseStringList(result) +} + // encode an object as base 64, serialize and then base64 encode func ObjToB64(obj interface{}) (string, bool) { b := bytes.Buffer{} diff --git a/misc_test.go b/misc_test.go index 2e82a68c..e6fb18f5 100644 --- a/misc_test.go +++ b/misc_test.go @@ -19,6 +19,7 @@ package main import ( "fmt" + "reflect" "testing" ) @@ -53,6 +54,13 @@ func TestMiscT1(t *testing.T) { func TestMiscT2(t *testing.T) { // TODO: compare the output with the actual list + p0 := "/" + r0 := []string{""} // TODO: is this correct? + if len(PathSplit(p0)) != len(r0) { + t.Errorf("Result should be: %q.", r0) + t.Errorf("Result should have a length of: %v.", len(r0)) + } + p1 := "/foo/bar/baz" r1 := []string{"", "foo", "bar", "baz"} if len(PathSplit(p1)) != len(r1) { @@ -198,3 +206,22 @@ func TestMiscT7(t *testing.T) { t.Errorf("Strings should match.") } } + +func TestMiscT8(t *testing.T) { + + r0 := []string{"/"} + if fullList0 := PathSplitFullReversed("/"); !reflect.DeepEqual(r0, fullList0) { + t.Errorf("PathSplitFullReversed expected: %v; got: %v.", r0, fullList0) + } + + r1 := []string{"/foo/bar/baz/file", "/foo/bar/baz/", "/foo/bar/", "/foo/", "/"} + if fullList1 := PathSplitFullReversed("/foo/bar/baz/file"); !reflect.DeepEqual(r1, fullList1) { + t.Errorf("PathSplitFullReversed expected: %v; got: %v.", r1, fullList1) + } + + r2 := []string{"/foo/bar/baz/dir/", "/foo/bar/baz/", "/foo/bar/", "/foo/", "/"} + if fullList2 := PathSplitFullReversed("/foo/bar/baz/dir/"); !reflect.DeepEqual(r2, fullList2) { + t.Errorf("PathSplitFullReversed expected: %v; got: %v.", r2, fullList2) + } + +} diff --git a/packagekit.go b/packagekit.go index 2801e169..69a6d086 100644 --- a/packagekit.go +++ b/packagekit.go @@ -462,6 +462,71 @@ loop: return nil } +// get the list of files that are contained inside a list of packageids +func (bus *Conn) GetFilesByPackageId(packageIds []string) (files map[string][]string, err error) { + ch := make(chan *dbus.Signal, PkBufferSize) // we need to buffer :( + interfacePath, err := bus.CreateTransaction() + if err != nil { + return + } + + var signals = []string{"Files", "ErrorCode", "Finished", "Destroy"} // "ItemProgress", "Status" ? + bus.matchSignal(ch, interfacePath, PkIfaceTransaction, signals) + + obj := bus.GetBus().Object(PkIface, interfacePath) // pass in found transaction path + call := obj.Call(FmtTransactionMethod("GetFiles"), 0, packageIds) + if call.Err != nil { + err = call.Err + return + } + files = make(map[string][]string) +loop: + for { + // FIXME: add a timeout option to error in case signals are dropped! + select { + case signal := <-ch: + + if signal.Path != interfacePath { + log.Println("PackageKit: Some wires have been crossed!") + continue loop + } + + if signal.Name == FmtTransactionMethod("ErrorCode") { + err = errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return + + // one signal returned per packageId found... + } else if signal.Name == FmtTransactionMethod("Files") { + if len(signal.Body) != 2 { // bad data + continue loop + } + var ok bool + var key string + var fileList []string + if key, ok = signal.Body[0].(string); !ok { + continue loop + } + if fileList, ok = signal.Body[1].([]string); !ok { + continue loop // failed conversion + } + files[key] = fileList // build up map + + continue loop + } else if signal.Name == FmtTransactionMethod("Finished") { + // TODO: should we wait for the Destroy signal? + break loop + } else if signal.Name == FmtTransactionMethod("Destroy") { + // should already be broken + break loop + } else { + err = errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return + } + } + } + return +} + // does flag exist inside data portion of packageId field? func FlagInData(flag, data string) bool { flags := strings.Split(data, ":") diff --git a/pkg.go b/pkg.go index 2ae31c2c..a1706fa4 100644 --- a/pkg.go +++ b/pkg.go @@ -278,6 +278,10 @@ func (obj *PkgRes) CheckApply(apply bool) (stateok bool, err error) { return false, nil // success } +func (obj *PkgRes) AutoEdges() AutoEdge { + return nil +} + func (obj *PkgRes) Compare(res Res) bool { switch res.(type) { case *PkgRes: diff --git a/resources.go b/resources.go index b7d1bce1..77384246 100644 --- a/resources.go +++ b/resources.go @@ -42,12 +42,40 @@ const ( resConvergedTimeout ) +// a unique identifier for a resource, namely it's name, and the kind ("type") +type ResUUID interface { + GetName() string + Kind() string + IFF(ResUUID) bool + + Reversed() bool // true means this resource happens before the generator +} + +type BaseUUID struct { + name string + kind string + + reversed *bool // piggyback edge information here +} + +type AutoEdge interface { + Next() []ResUUID // call to get list of edges to add + Test([]bool) bool // call until false +} + +type MetaParams struct { + AutoEdge bool `yaml:"autoedge"` // metaparam, should we generate auto edges? // XXX should default to true +} + type Res interface { Init() - GetName() string // can't be named "Name()" because of struct field + GetName() string // can't be named "Name()" because of struct field + GetUUIDs() []ResUUID // most resources only return one + GetMeta() MetaParams Kind() string Watch() CheckApply(bool) (bool, error) + AutoEdges() AutoEdge SetVertex(*Vertex) SetConvergedCallback(ctimeout int, converged chan bool) Compare(Res) bool @@ -66,8 +94,9 @@ type Res interface { } type BaseRes struct { - Name string `yaml:"name"` - timestamp int64 // last updated timestamp ? + Name string `yaml:"name"` + Meta MetaParams `yaml:"meta"` // struct of all the metaparams + timestamp int64 // last updated timestamp ? events chan Event vertex *Vertex state resState @@ -95,6 +124,43 @@ func NewNoopRes(name string) *NoopRes { } } +// wraps the IFF method when used with a list of UUID's +func UUIDExistsInUUIDs(uuid ResUUID, uuids []ResUUID) bool { + for _, u := range uuids { + if uuid.IFF(u) { + return true + } + } + return false +} + +func (obj *BaseUUID) GetName() string { + return obj.name +} + +func (obj *BaseUUID) Kind() string { + return obj.kind +} + +// if and only if they are equivalent, return true +// if they are not equivalent, return false +// most resource will want to override this method, since it does the important +// work of actually discerning if two resources are identical in function +func (obj *BaseUUID) IFF(uuid ResUUID) bool { + res, ok := uuid.(*BaseUUID) + if !ok { + return false + } + return obj.name == res.name +} + +func (obj *BaseUUID) Reversed() bool { + if obj.reversed == nil { + log.Fatal("Programming error!") + } + return *obj.reversed +} + // initialize structures like channels if created without New constructor func (obj *BaseRes) Init() { obj.events = make(chan Event) @@ -109,6 +175,10 @@ func (obj *BaseRes) Kind() string { return "Base" } +func (obj *BaseRes) GetMeta() MetaParams { + return obj.Meta +} + func (obj *BaseRes) GetVertex() *Vertex { return obj.vertex } @@ -392,6 +462,23 @@ func (obj *NoopRes) CheckApply(apply bool) (stateok bool, err error) { return true, nil // state is always okay } +type NoopUUID struct { + BaseUUID +} + +func (obj *NoopRes) AutoEdges() AutoEdge { + return nil +} + +// include all params to make a unique identification of this object +// most resources only return one +func (obj *NoopRes) GetUUIDs() []ResUUID { + x := &NoopUUID{ + BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()}, + } + return []ResUUID{x} +} + func (obj *NoopRes) Compare(res Res) bool { switch res.(type) { // we can only compare NoopRes to others of the same resource diff --git a/svc.go b/svc.go index e5c8da92..adb3ea7b 100644 --- a/svc.go +++ b/svc.go @@ -308,6 +308,33 @@ func (obj *SvcRes) CheckApply(apply bool) (stateok bool, err error) { return false, nil // success } +type SvcUUID struct { + BaseUUID +} + +// if and only if they are equivalent, return true +// if they are not equivalent, return false +func (obj *SvcUUID) IFF(uuid ResUUID) bool { + res, ok := uuid.(*SvcUUID) + if !ok { + return false + } + return obj.name == res.name +} + +func (obj *SvcRes) AutoEdges() AutoEdge { + // TODO: add auto edges to the files that provide the service files, + // which might come from a pkg resource perhaps! + return nil +} + +// include all params to make a unique identification of this object +func (obj *SvcRes) GetUUIDs() []ResUUID { + x := &SvcUUID{BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()}} + return []ResUUID{x} + +} + func (obj *SvcRes) Compare(res Res) bool { switch res.(type) { case *SvcRes: