10b8c93da4
By adding the "kind" to the base resource, it is still identifiable even when the resource specific elements are gone in calls that are only defined in the base resource. This is also more logical for building resources! This also switches resources to use an Init() method. This will be useful for when resources have more complex initialization to do.
326 lines
9.8 KiB
Go
326 lines
9.8 KiB
Go
// Mgmt
|
|
// Copyright (C) 2013-2016+ James Shubin and the project contributors
|
|
// Written by James Shubin <james@shubin.ca> and the project contributors
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package main
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"gopkg.in/yaml.v2"
|
|
"io/ioutil"
|
|
"log"
|
|
"strings"
|
|
)
|
|
|
|
type collectorResConfig struct {
|
|
Res string `yaml:"res"`
|
|
Pattern string `yaml:"pattern"` // XXX: Not Implemented
|
|
}
|
|
|
|
type vertexConfig struct {
|
|
Res string `yaml:"res"`
|
|
Name string `yaml:"name"`
|
|
}
|
|
|
|
type edgeConfig struct {
|
|
Name string `yaml:"name"`
|
|
From vertexConfig `yaml:"from"`
|
|
To vertexConfig `yaml:"to"`
|
|
}
|
|
|
|
type GraphConfig struct {
|
|
Graph string `yaml:"graph"`
|
|
Resources struct {
|
|
Noop []*NoopRes `yaml:"noop"`
|
|
Pkg []*PkgRes `yaml:"pkg"`
|
|
File []*FileRes `yaml:"file"`
|
|
Svc []*SvcRes `yaml:"svc"`
|
|
Exec []*ExecRes `yaml:"exec"`
|
|
} `yaml:"resources"`
|
|
Collector []collectorResConfig `yaml:"collect"`
|
|
Edges []edgeConfig `yaml:"edges"`
|
|
Comment string `yaml:"comment"`
|
|
}
|
|
|
|
func (c *GraphConfig) Parse(data []byte) error {
|
|
if err := yaml.Unmarshal(data, c); err != nil {
|
|
return err
|
|
}
|
|
if c.Graph == "" {
|
|
return errors.New("Graph config: invalid `graph`")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ParseConfigFromFile(filename string) *GraphConfig {
|
|
data, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
log.Printf("Error: Config: ParseConfigFromFile: File: %v", err)
|
|
return nil
|
|
}
|
|
|
|
var config GraphConfig
|
|
if err := config.Parse(data); err != nil {
|
|
log.Printf("Error: Config: ParseConfigFromFile: Parse: %v", err)
|
|
return nil
|
|
}
|
|
|
|
return &config
|
|
}
|
|
|
|
// XXX: we need to fix this function so that it either fails without modifying
|
|
// the graph, passes successfully and modifies it, or basically panics i guess
|
|
// this way an invalid compilation can leave the old graph running, and we we
|
|
// don't modify a partial graph. so we really need to validate, and then perform
|
|
// whatever actions are necessary
|
|
// finding some way to do this on a copy of the graph, and then do a graph diff
|
|
// and merge the new data into the old graph would be more appropriate, in
|
|
// particular if we can ensure the graph merge can't fail. As for the putting
|
|
// of stuff into etcd, we should probably store the operations to complete in
|
|
// the new graph, and keep retrying until it succeeds, thus blocking any new
|
|
// etcd operations until that time.
|
|
func UpdateGraphFromConfig(config *GraphConfig, hostname string, g *Graph, etcdO *EtcdWObject) bool {
|
|
|
|
var NoopMap = make(map[string]*Vertex)
|
|
var PkgMap = make(map[string]*Vertex)
|
|
var FileMap = make(map[string]*Vertex)
|
|
var SvcMap = make(map[string]*Vertex)
|
|
var ExecMap = make(map[string]*Vertex)
|
|
|
|
var lookup = make(map[string]map[string]*Vertex)
|
|
lookup["noop"] = NoopMap
|
|
lookup["pkg"] = PkgMap
|
|
lookup["file"] = FileMap
|
|
lookup["svc"] = SvcMap
|
|
lookup["exec"] = ExecMap
|
|
|
|
//log.Printf("%+v", config) // debug
|
|
|
|
g.SetName(config.Graph) // set graph name
|
|
|
|
var keep []*Vertex // list of vertex which are the same in new graph
|
|
|
|
for _, obj := range config.Resources.Noop {
|
|
v := g.GetVertexMatch(obj)
|
|
if v == nil { // no match found
|
|
obj.Init()
|
|
v = NewVertex(obj)
|
|
g.AddVertex(v) // call standalone in case not part of an edge
|
|
}
|
|
NoopMap[obj.Name] = v // used for constructing edges
|
|
keep = append(keep, v) // append
|
|
}
|
|
|
|
for _, obj := range config.Resources.Pkg {
|
|
v := g.GetVertexMatch(obj)
|
|
if v == nil { // no match found
|
|
obj.Init()
|
|
v = NewVertex(obj)
|
|
g.AddVertex(v) // call standalone in case not part of an edge
|
|
}
|
|
PkgMap[obj.Name] = v // used for constructing edges
|
|
keep = append(keep, v) // append
|
|
}
|
|
|
|
for _, obj := range config.Resources.File {
|
|
// XXX: should we export based on a @@ prefix, or a metaparam
|
|
// like exported => true || exported => (host pattern)||(other pattern?)
|
|
if strings.HasPrefix(obj.Name, "@@") { // exported resource
|
|
// add to etcd storage...
|
|
obj.Name = obj.Name[2:] //slice off @@
|
|
if !etcdO.EtcdPut(hostname, obj.Name, "file", obj) {
|
|
log.Printf("Problem exporting file resource %v.", obj.Name)
|
|
continue
|
|
}
|
|
} else {
|
|
// XXX: we don't have a way of knowing if any of the
|
|
// metaparams are undefined, and as a result to set the
|
|
// defaults that we want! I hate the go yaml parser!!!
|
|
v := g.GetVertexMatch(obj)
|
|
if v == nil { // no match found
|
|
obj.Init()
|
|
v = NewVertex(obj)
|
|
g.AddVertex(v) // call standalone in case not part of an edge
|
|
}
|
|
FileMap[obj.Name] = v // used for constructing edges
|
|
keep = append(keep, v) // append
|
|
}
|
|
}
|
|
|
|
for _, obj := range config.Resources.Svc {
|
|
v := g.GetVertexMatch(obj)
|
|
if v == nil { // no match found
|
|
obj.Init()
|
|
v = NewVertex(obj)
|
|
g.AddVertex(v) // call standalone in case not part of an edge
|
|
}
|
|
SvcMap[obj.Name] = v // used for constructing edges
|
|
keep = append(keep, v) // append
|
|
}
|
|
|
|
for _, obj := range config.Resources.Exec {
|
|
v := g.GetVertexMatch(obj)
|
|
if v == nil { // no match found
|
|
obj.Init()
|
|
v = NewVertex(obj)
|
|
g.AddVertex(v) // call standalone in case not part of an edge
|
|
}
|
|
ExecMap[obj.Name] = v // used for constructing edges
|
|
keep = append(keep, v) // append
|
|
}
|
|
|
|
// lookup from etcd graph
|
|
// do all the graph look ups in one single step, so that if the etcd
|
|
// database changes, we don't have a partial state of affairs...
|
|
nodes, ok := etcdO.EtcdGet()
|
|
if ok {
|
|
for _, t := range config.Collector {
|
|
// XXX: use t.Res and optionally t.Pattern to collect from etcd storage
|
|
log.Printf("Collect: %v; Pattern: %v", t.Res, t.Pattern)
|
|
|
|
for _, x := range etcdO.EtcdGetProcess(nodes, "file") {
|
|
var obj *FileRes
|
|
if B64ToObj(x, &obj) != true {
|
|
log.Printf("Collect: File: %v not collected!", x)
|
|
continue
|
|
}
|
|
if t.Pattern != "" { // XXX: currently the pattern for files can only override the Dirname variable :P
|
|
obj.Dirname = t.Pattern
|
|
}
|
|
|
|
log.Printf("Collect: File: %v collected!", obj.GetName())
|
|
|
|
// XXX: similar to file add code:
|
|
v := g.GetVertexMatch(obj)
|
|
if v == nil { // no match found
|
|
obj.Init() // initialize go channels or things won't work!!!
|
|
v = NewVertex(obj)
|
|
g.AddVertex(v) // call standalone in case not part of an edge
|
|
}
|
|
FileMap[obj.GetName()] = v // used for constructing edges
|
|
keep = append(keep, v) // append
|
|
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
// get rid of any vertices we shouldn't "keep" (that aren't in new graph)
|
|
for _, v := range g.GetVertices() {
|
|
if !HasVertex(v, keep) {
|
|
// wait for exit before starting new graph!
|
|
v.Res.SendEvent(eventExit, true, false)
|
|
g.DeleteVertex(v)
|
|
}
|
|
}
|
|
|
|
for _, e := range config.Edges {
|
|
if _, ok := lookup[e.From.Res]; !ok {
|
|
return false
|
|
}
|
|
if _, ok := lookup[e.To.Res]; !ok {
|
|
return false
|
|
}
|
|
if _, ok := lookup[e.From.Res][e.From.Name]; !ok {
|
|
return false
|
|
}
|
|
if _, ok := lookup[e.To.Res][e.To.Name]; !ok {
|
|
return false
|
|
}
|
|
g.AddEdge(lookup[e.From.Res][e.From.Name], lookup[e.To.Res][e.To.Name], NewEdge(e.Name))
|
|
}
|
|
|
|
// add auto edges
|
|
log.Println("Compile: Adding AutoEdges...")
|
|
for _, v := range g.GetVertices() { // for each vertexes autoedges
|
|
if !v.GetMeta().AutoEdge { // is the metaparam true?
|
|
continue
|
|
}
|
|
autoEdgeObj := v.AutoEdges()
|
|
if autoEdgeObj == nil {
|
|
log.Printf("%v[%v]: Config: No auto edges were found!", v.Kind(), v.GetName())
|
|
continue // next vertex
|
|
}
|
|
|
|
for { // while the autoEdgeObj has more uuids to add...
|
|
uuids := autoEdgeObj.Next() // get some!
|
|
if uuids == nil {
|
|
log.Printf("%v[%v]: Config: The auto edge list is empty!", v.Kind(), v.GetName())
|
|
break // inner loop
|
|
}
|
|
if DEBUG {
|
|
log.Println("Compile: AutoEdge: UUIDS:")
|
|
for i, u := range uuids {
|
|
log.Printf("Compile: AutoEdge: UUID%d: %v", i, u)
|
|
}
|
|
}
|
|
|
|
// match and add edges
|
|
result := g.AddEdgesByMatchingUUIDS(v, uuids)
|
|
|
|
// report back, and find out if we should continue
|
|
if !autoEdgeObj.Test(result) {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
// add edges to the vertex in a graph based on if it matches a uuid list
|
|
func (g *Graph) AddEdgesByMatchingUUIDS(v *Vertex, uuids []ResUUID) []bool {
|
|
// search for edges and see what matches!
|
|
var result []bool
|
|
|
|
// loop through each uuid, and see if it matches any vertex
|
|
for _, uuid := range uuids {
|
|
var found = false
|
|
// uuid is a ResUUID object
|
|
for _, vv := range g.GetVertices() { // search
|
|
if v == vv { // skip self
|
|
continue
|
|
}
|
|
if DEBUG {
|
|
log.Printf("Compile: AutoEdge: Match: %v[%v] with UUID: %v[%v]", vv.Kind(), vv.GetName(), uuid.Kind(), uuid.GetName())
|
|
}
|
|
// we must match to an effective UUID for the resource,
|
|
// that is to say, the name value of a res is a helpful
|
|
// handle, but it is not necessarily a unique identity!
|
|
// remember, resources can return multiple UUID's each!
|
|
if UUIDExistsInUUIDs(uuid, vv.GetUUIDs()) {
|
|
// add edge from: vv -> v
|
|
if uuid.Reversed() {
|
|
txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", vv.Kind(), vv.GetName(), v.Kind(), v.GetName())
|
|
log.Printf("Compile: Adding %v", txt)
|
|
g.AddEdge(vv, v, NewEdge(txt))
|
|
} else { // edges go the "normal" way, eg: pkg resource
|
|
txt := fmt.Sprintf("AutoEdge: %v[%v] -> %v[%v]", v.Kind(), v.GetName(), vv.Kind(), vv.GetName())
|
|
log.Printf("Compile: Adding %v", txt)
|
|
g.AddEdge(v, vv, NewEdge(txt))
|
|
}
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
|
|
result = append(result, found)
|
|
}
|
|
|
|
return result
|
|
}
|