// Mgmt // Copyright (C) 2013-2017+ James Shubin and the project contributors // Written by James Shubin and the project contributors // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . package resources import ( "fmt" "log" "github.com/purpleidea/mgmt/pgraph" "github.com/purpleidea/mgmt/util" errwrap "github.com/pkg/errors" ) // AutoGrouper is the required interface to implement for an autogroup algorithm. type AutoGrouper interface { // listed in the order these are typically called in... name() string // friendly identifier init(*pgraph.Graph) error // only call once vertexNext() (pgraph.Vertex, pgraph.Vertex, error) // mostly algorithmic vertexCmp(pgraph.Vertex, pgraph.Vertex) error // can we merge these ? vertexMerge(pgraph.Vertex, pgraph.Vertex) (pgraph.Vertex, error) // vertex merge fn to use edgeMerge(pgraph.Edge, pgraph.Edge) pgraph.Edge // edge merge fn to use vertexTest(bool) (bool, error) // call until false } // baseGrouper is the base type for implementing the AutoGrouper interface. type baseGrouper struct { graph *pgraph.Graph // store a pointer to the graph vertices []pgraph.Vertex // cached list of vertices i int j int done bool } // name provides a friendly name for the logs to see. func (ag *baseGrouper) name() string { return "baseGrouper" } // init is called only once and before using other AutoGrouper interface methods // the name method is the only exception: call it any time without side effects! func (ag *baseGrouper) init(g *pgraph.Graph) error { if ag.graph != nil { return fmt.Errorf("the init method has already been called") } ag.graph = g // pointer ag.vertices = ag.graph.VerticesSorted() // cache in deterministic order! ag.i = 0 ag.j = 0 if len(ag.vertices) == 0 { // empty graph ag.done = true return nil } return nil } // vertexNext is a simple iterator that loops through vertex (pair) combinations // an intelligent algorithm would selectively offer only valid pairs of vertices // these should satisfy logical grouping requirements for the autogroup designs! // the desired algorithms can override, but keep this method as a base iterator! func (ag *baseGrouper) vertexNext() (v1, v2 pgraph.Vertex, err error) { // this does a for v... { for w... { return v, w }} but stepwise! l := len(ag.vertices) if ag.i < l { v1 = ag.vertices[ag.i] } if ag.j < l { v2 = ag.vertices[ag.j] } // in case the vertex was deleted if !ag.graph.HasVertex(v1) { v1 = nil } if !ag.graph.HasVertex(v2) { v2 = nil } // two nested loops... if ag.j < l { ag.j++ } if ag.j == l { ag.j = 0 if ag.i < l { ag.i++ } if ag.i == l { ag.done = true } } return } func (ag *baseGrouper) vertexCmp(v1, v2 pgraph.Vertex) error { if v1 == nil || v2 == nil { return fmt.Errorf("the vertex is nil") } if v1 == v2 { // skip yourself return fmt.Errorf("the vertices are the same") } if VtoR(v1).GetKind() != VtoR(v2).GetKind() { // we must group similar kinds // TODO: maybe future resources won't need this limitation? return fmt.Errorf("the two resources aren't the same kind") } // someone doesn't want to group! if !VtoR(v1).Meta().AutoGroup || !VtoR(v2).Meta().AutoGroup { return fmt.Errorf("one of the autogroup flags is false") } if VtoR(v1).IsGrouped() { // already grouped! return fmt.Errorf("already grouped") } if len(VtoR(v2).GetGroup()) > 0 { // already has children grouped! return fmt.Errorf("already has groups") } if !VtoR(v1).GroupCmp(VtoR(v2)) { // resource groupcmp failed! return fmt.Errorf("the GroupCmp failed") } return nil // success } func (ag *baseGrouper) vertexMerge(v1, v2 pgraph.Vertex) (v pgraph.Vertex, err error) { // NOTE: it's important to use w.Res instead of w, b/c // the w by itself is the *Vertex obj, not the *Res obj // which is contained within it! They both satisfy the // Res interface, which is why both will compile! :( err = VtoR(v1).GroupRes(VtoR(v2)) // GroupRes skips stupid groupings return // success or fail, and no need to merge the actual vertices! } func (ag *baseGrouper) edgeMerge(e1, e2 pgraph.Edge) pgraph.Edge { // FIXME: should we merge the edge.Notify or edge.refresh values? return e1 // noop } // vertexTest processes the results of the grouping for the algorithm to know // return an error if something went horribly wrong, and bool false to stop. func (ag *baseGrouper) vertexTest(b bool) (bool, error) { // NOTE: this particular baseGrouper version doesn't track what happens // because since we iterate over every pair, we don't care which merge! if ag.done { return false, nil } return true, nil } // NonReachabilityGrouper is the most straight-forward algorithm for grouping. // TODO: this algorithm may not be correct in all cases. replace if needed! type NonReachabilityGrouper struct { baseGrouper // "inherit" what we want, and reimplement the rest } func (ag *NonReachabilityGrouper) name() string { return "NonReachabilityGrouper" } // This algorithm relies on the observation that if there's a path from a to b, // then they *can't* be merged (b/c of the existing dependency) so therefore we // merge anything that *doesn't* satisfy this condition or that of the reverse! func (ag *NonReachabilityGrouper) vertexNext() (v1, v2 pgraph.Vertex, err error) { for { v1, v2, err = ag.baseGrouper.vertexNext() // get all iterable pairs if err != nil { log.Fatalf("error running autoGroup(vertexNext): %v", err) } if v1 != v2 { // ignore self cmp early (perf optimization) // if NOT reachable, they're viable... out1 := ag.graph.Reachability(v1, v2) out2 := ag.graph.Reachability(v2, v1) if len(out1) == 0 && len(out2) == 0 { return // return v1 and v2, they're viable } } // if we got here, it means we're skipping over this candidate! if ok, err := ag.baseGrouper.vertexTest(false); err != nil { log.Fatalf("error running autoGroup(vertexTest): %v", err) } else if !ok { return nil, nil, nil // done! } // the vertexTest passed, so loop and try with a new pair... } } // VertexMerge merges v2 into v1 by reattaching the edges where appropriate, // and then by deleting v2 from the graph. Since more than one edge between two // vertices is not allowed, duplicate edges are merged as well. an edge merge // function can be provided if you'd like to control how you merge the edges! func VertexMerge(g *pgraph.Graph, v1, v2 pgraph.Vertex, vertexMergeFn func(pgraph.Vertex, pgraph.Vertex) (pgraph.Vertex, error), edgeMergeFn func(pgraph.Edge, pgraph.Edge) pgraph.Edge) error { // methodology // 1) edges between v1 and v2 are removed //Loop: for k1 := range g.Adjacency() { for k2 := range g.Adjacency()[k1] { // v1 -> v2 || v2 -> v1 if (k1 == v1 && k2 == v2) || (k1 == v2 && k2 == v1) { delete(g.Adjacency()[k1], k2) // delete map & edge // NOTE: if we assume this is a DAG, then we can // assume only v1 -> v2 OR v2 -> v1 exists, and // we can break out of these loops immediately! //break Loop break } } } // 2) edges that point towards v2 from X now point to v1 from X (no dupes) for _, x := range g.IncomingGraphVertices(v2) { // all to vertex v (??? -> v) e := g.Adjacency()[x][v2] // previous edge r := g.Reachability(x, v1) // merge e with ex := g.Adjacency()[x][v1] if it exists! if ex, exists := g.Adjacency()[x][v1]; exists && edgeMergeFn != nil && len(r) == 0 { e = edgeMergeFn(e, ex) } if len(r) == 0 { // if not reachable, add it g.AddEdge(x, v1, e) // overwrite edge } else if edgeMergeFn != nil { // reachable, merge e through... prev := x // initial condition for i, next := range r { if i == 0 { // next == prev, therefore skip continue } // this edge is from: prev, to: next ex, _ := g.Adjacency()[prev][next] // get ex = edgeMergeFn(ex, e) g.Adjacency()[prev][next] = ex // set prev = next } } delete(g.Adjacency()[x], v2) // delete old edge } // 3) edges that point from v2 to X now point from v1 to X (no dupes) for _, x := range g.OutgoingGraphVertices(v2) { // all from vertex v (v -> ???) e := g.Adjacency()[v2][x] // previous edge r := g.Reachability(v1, x) // merge e with ex := g.Adjacency()[v1][x] if it exists! if ex, exists := g.Adjacency()[v1][x]; exists && edgeMergeFn != nil && len(r) == 0 { e = edgeMergeFn(e, ex) } if len(r) == 0 { g.AddEdge(v1, x, e) // overwrite edge } else if edgeMergeFn != nil { // reachable, merge e through... prev := v1 // initial condition for i, next := range r { if i == 0 { // next == prev, therefore skip continue } // this edge is from: prev, to: next ex, _ := g.Adjacency()[prev][next] ex = edgeMergeFn(ex, e) g.Adjacency()[prev][next] = ex prev = next } } delete(g.Adjacency()[v2], x) } // 4) merge and then remove the (now merged/grouped) vertex if vertexMergeFn != nil { // run vertex merge function if v, err := vertexMergeFn(v1, v2); err != nil { return err } else if v != nil { // replace v1 with the "merged" version... //*v1 = *v // TODO: is this safe? (replacing mutexes is undefined!) v1 = v } } g.DeleteVertex(v2) // remove grouped vertex // 5) creation of a cyclic graph should throw an error if _, err := g.TopologicalSort(); err != nil { // am i a dag or not? return errwrap.Wrapf(err, "the TopologicalSort failed") // not a dag } return nil // success } // autoGroup is the mechanical auto group "runner" that runs the interface spec. func autoGroup(g *pgraph.Graph, ag AutoGrouper) chan string { strch := make(chan string) // output log messages here go func(strch chan string) { strch <- fmt.Sprintf("Compile: Grouping: Algorithm: %v...", ag.name()) if err := ag.init(g); err != nil { log.Fatalf("error running autoGroup(init): %v", err) } for { var v, w pgraph.Vertex v, w, err := ag.vertexNext() // get pair to compare if err != nil { log.Fatalf("error running autoGroup(vertexNext): %v", err) } merged := false // save names since they change during the runs vStr := fmt.Sprintf("%s", v) // valid even if it is nil wStr := fmt.Sprintf("%s", w) if err := ag.vertexCmp(v, w); err != nil { // cmp ? if b, ok := g.Value("debug"); ok && util.Bool(b) { strch <- fmt.Sprintf("Compile: Grouping: !GroupCmp for: %s into %s", wStr, vStr) } // remove grouped vertex and merge edges (res is safe) } else if err := VertexMerge(g, v, w, ag.vertexMerge, ag.edgeMerge); err != nil { // merge... strch <- fmt.Sprintf("Compile: Grouping: !VertexMerge for: %s into %s", wStr, vStr) } else { // success! strch <- fmt.Sprintf("Compile: Grouping: Success for: %s into %s", wStr, vStr) merged = true // woo } // did these get used? if ok, err := ag.vertexTest(merged); err != nil { log.Fatalf("error running autoGroup(vertexTest): %v", err) } else if !ok { break // done! } } close(strch) return }(strch) // call function return strch } // AutoGroup runs the auto grouping on the graph and prints out log messages. func AutoGroup(g *pgraph.Graph, ag AutoGrouper) { // receive log messages from channel... // this allows test cases to avoid printing them when they're unwanted! // TODO: this algorithm may not be correct in all cases. replace if needed! for str := range autoGroup(g, ag) { log.Println(str) } }