// Mgmt // Copyright (C) 2013-2016+ James Shubin and the project contributors // Written by James Shubin and the project contributors // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package main import ( "errors" "fmt" "gopkg.in/yaml.v2" "io/ioutil" "log" "strings" ) type collectorResConfig struct { Res string `yaml:"res"` Pattern string `yaml:"pattern"` // XXX: Not Implemented } type vertexConfig struct { Res string `yaml:"res"` Name string `yaml:"name"` } type edgeConfig struct { Name string `yaml:"name"` From vertexConfig `yaml:"from"` To vertexConfig `yaml:"to"` } type GraphConfig struct { Graph string `yaml:"graph"` Resources struct { Noop []*NoopRes `yaml:"noop"` Pkg []*PkgRes `yaml:"pkg"` File []*FileRes `yaml:"file"` Svc []*SvcRes `yaml:"svc"` Exec []*ExecRes `yaml:"exec"` } `yaml:"resources"` Collector []collectorResConfig `yaml:"collect"` Edges []edgeConfig `yaml:"edges"` Comment string `yaml:"comment"` } func (c *GraphConfig) Parse(data []byte) error { if err := yaml.Unmarshal(data, c); err != nil { return err } if c.Graph == "" { return errors.New("Graph config: invalid `graph`") } return nil } func ParseConfigFromFile(filename string) *GraphConfig { data, err := ioutil.ReadFile(filename) if err != nil { log.Printf("Error: Config: ParseConfigFromFile: File: %v", err) return nil } var config GraphConfig if err := config.Parse(data); err != nil { log.Printf("Error: Config: ParseConfigFromFile: Parse: %v", err) return nil } return &config } // XXX: we need to fix this function so that it either fails without modifying // the graph, passes successfully and modifies it, or basically panics i guess // this way an invalid compilation can leave the old graph running, and we we // don't modify a partial graph. so we really need to validate, and then perform // whatever actions are necessary // finding some way to do this on a copy of the graph, and then do a graph diff // and merge the new data into the old graph would be more appropriate, in // particular if we can ensure the graph merge can't fail. As for the putting // of stuff into etcd, we should probably store the operations to complete in // the new graph, and keep retrying until it succeeds, thus blocking any new // etcd operations until that time. func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO *EtcdWObject) bool { var NoopMap = make(map[string]*Vertex) var PkgMap = make(map[string]*Vertex) var FileMap = make(map[string]*Vertex) var SvcMap = make(map[string]*Vertex) var ExecMap = make(map[string]*Vertex) var lookup = make(map[string]map[string]*Vertex) lookup["noop"] = NoopMap lookup["pkg"] = PkgMap lookup["file"] = FileMap lookup["svc"] = SvcMap lookup["exec"] = ExecMap //log.Printf("%+v", config) // debug g.SetName(config.Graph) // set graph name var keep []*Vertex // list of vertex which are the same in new graph for _, obj := range config.Resources.Noop { v := g.GetVertexMatch(obj) if v == nil { // no match found obj.Init() v = NewVertex(obj) g.AddVertex(v) // call standalone in case not part of an edge } NoopMap[obj.Name] = v // used for constructing edges keep = append(keep, v) // append } for _, obj := range config.Resources.Pkg { v := g.GetVertexMatch(obj) if v == nil { // no match found obj.Init() v = NewVertex(obj) g.AddVertex(v) // call standalone in case not part of an edge } PkgMap[obj.Name] = v // used for constructing edges keep = append(keep, v) // append } for _, obj := range config.Resources.File { // XXX: should we export based on a @@ prefix, or a metaparam // like exported => true || exported => (host pattern)||(other pattern?) if strings.HasPrefix(obj.Name, "@@") { // exported resource // add to etcd storage... obj.Name = obj.Name[2:] //slice off @@ if !etcdO.EtcdPut(hostname, obj.Name, "file", obj) { log.Printf("Problem exporting file resource %v.", obj.Name) continue } } else { // XXX: we don't have a way of knowing if any of the // metaparams are undefined, and as a result to set the // defaults that we want! I hate the go yaml parser!!! v := g.GetVertexMatch(obj) if v == nil { // no match found obj.Init() v = NewVertex(obj) g.AddVertex(v) // call standalone in case not part of an edge } FileMap[obj.Name] = v // used for constructing edges keep = append(keep, v) // append } } for _, obj := range config.Resources.Svc { v := g.GetVertexMatch(obj) if v == nil { // no match found obj.Init() v = NewVertex(obj) g.AddVertex(v) // call standalone in case not part of an edge } SvcMap[obj.Name] = v // used for constructing edges keep = append(keep, v) // append } for _, obj := range config.Resources.Exec { v := g.GetVertexMatch(obj) if v == nil { // no match found obj.Init() v = NewVertex(obj) g.AddVertex(v) // call standalone in case not part of an edge } ExecMap[obj.Name] = v // used for constructing edges keep = append(keep, v) // append } // lookup from etcd graph // do all the graph look ups in one single step, so that if the etcd // database changes, we don't have a partial state of affairs... nodes, ok := etcdO.EtcdGet() if ok { for _, t := range config.Collector { // XXX: use t.Res and optionally t.Pattern to collect from etcd storage log.Printf("Collect: %v; Pattern: %v", t.Res, t.Pattern) for _, x := range etcdO.EtcdGetProcess(nodes, "file") { var obj *FileRes if B64ToObj(x, &obj) != true { log.Printf("Collect: File: %v not collected!", x) continue } if t.Pattern != "" { // XXX: currently the pattern for files can only override the Dirname variable :P obj.Dirname = t.Pattern } log.Printf("Collect: File: %v collected!", obj.GetName()) // XXX: similar to file add code: v := g.GetVertexMatch(obj) if v == nil { // no match found obj.Init() // initialize go channels or things won't work!!! v = NewVertex(obj) g.AddVertex(v) // call standalone in case not part of an edge } FileMap[obj.GetName()] = v // used for constructing edges keep = append(keep, v) // append } } } // get rid of any vertices we shouldn't "keep" (that aren't in new graph) for _, v := range g.GetVertices() { if !HasVertex(v, keep) { // wait for exit before starting new graph! v.Res.SendEvent(eventExit, true, false) g.DeleteVertex(v) } } for _, e := range config.Edges { if _, ok := lookup[e.From.Res]; !ok { return false } if _, ok := lookup[e.To.Res]; !ok { return false } if _, ok := lookup[e.From.Res][e.From.Name]; !ok { return false } if _, ok := lookup[e.To.Res][e.To.Name]; !ok { return false } g.AddEdge(lookup[e.From.Res][e.From.Name], lookup[e.To.Res][e.To.Name], NewEdge(e.Name)) } // add auto edges log.Println("Compile: Adding AutoEdges...") for _, v := range g.GetVertices() { // for each vertexes autoedges if !v.GetMeta().AutoEdge { // is the metaparam true? continue } autoEdgeObj := v.AutoEdges() if autoEdgeObj == nil { log.Printf("%v[%v]: Config: No auto edges were found!", v.Kind(), v.GetName()) continue // next vertex } for { // while the autoEdgeObj has more uuids to add... uuids := autoEdgeObj.Next() // get some! if uuids == nil { log.Printf("%v[%v]: Config: The auto edge list is empty!", v.Kind(), v.GetName()) break // inner loop } if DEBUG { log.Println("Compile: AutoEdge: UUIDS:") for i, u := range uuids { log.Printf("Compile: AutoEdge: UUID%d: %v", i, u) } } // match and add edges result := g.AddEdgesByMatchingUUIDS(v, uuids) // report back, and find out if we should continue if !autoEdgeObj.Test(result) { break } } } return true } // add edges to the vertex in a graph based on if it matches a uuid list func (g *Graph) AddEdgesByMatchingUUIDS(v *Vertex, uuids []ResUUID) []bool { // search for edges and see what matches! var result []bool // loop through each uuid, and see if it matches any vertex for _, uuid := range uuids { var found = false // uuid is a ResUUID object for _, vv := range g.GetVertices() { // search if v == vv { // skip self continue } if DEBUG { log.Printf("Compile: AutoEdge: Match: %v[%v] with UUID: %v[%v]", vv.Kind(), vv.GetName(), uuid.Kind(), uuid.GetName()) } // we must match to an effective UUID for the resource, // that is to say, the name value of a res is a helpful // handle, but it is not necessarily a unique identity! // remember, resources can return multiple UUID's each! if UUIDExistsInUUIDs(uuid, vv.GetUUIDs()) { // add edge from: vv -> v if uuid.Reversed() { txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", vv.Kind(), vv.GetName(), v.Kind(), v.GetName()) log.Printf("Compile: Adding %v", txt) g.AddEdge(vv, v, NewEdge(txt)) } else { // edges go the "normal" way, eg: pkg resource txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", v.Kind(), v.GetName(), vv.Kind(), vv.GetName()) log.Printf("Compile: Adding %v", txt) g.AddEdge(v, vv, NewEdge(txt)) } found = true break } } result = append(result, found) } return result }