lourenco/mgmt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

engine: Resources package rewrite

Browse Source 
This giant patch makes some much needed improvements to the code base.

* The engine has been rewritten and lives within engine/graph/
* All of the common interfaces and code now live in engine/
* All of the resources are in one package called engine/resources/
* The Res API can use different "traits" from engine/traits/
* The Res API has been simplified to hide many of the old internals
* The Watch & Process loops were previously inverted, but is now fixed
* The likelihood of package cycles has been reduced drastically
* And much, much more...

Unfortunately, some code had to be temporarily removed. The remote code
had to be taken out, as did the prometheus code. We hope to have these
back in new forms as soon as possible.
This commit is contained in:
James Shubin
2018-03-13 12:02:44 -04:00
parent ef49aa7e08
commit 9969286224
140 changed files with 9130 additions and 8764 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
yamlgraph/gapi.go
View File

@@ -19,12 +19,11 @@ package yamlgraph
import (
"fmt"
"log"
"sync"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/gapi"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
errwrap "github.com/pkg/errors"
"github.com/urfave/cli"
@@ -55,7 +54,7 @@ type GAPI struct {
// should take the prefix of the registered name. On activation, if there are
// any validation problems, you should return an error. If this was not
// activated, then you should return a nil GAPI and a nil error.
func (obj *GAPI) Cli(c *cli.Context, fs resources.Fs) (*gapi.Deploy, error) {
func (obj *GAPI) Cli(c *cli.Context, fs engine.Fs) (*gapi.Deploy, error) {
if s := c.String(Name); c.IsSet(Name) {
if s == "" {
return nil, fmt.Errorf("input yaml is empty")
@@ -120,9 +119,17 @@ func (obj *GAPI) Graph() (*pgraph.Graph, error) {
return nil, errwrap.Wrapf(err, "can't read yaml from file `%s`", Start)
}
config := ParseConfigFromFile(b)
debug := obj.data.Debug
logf := func(format string, v ...interface{}) {
// TODO: add the Name prefix in parent logger
obj.data.Logf(Name+": "+format, v...)
}
config, err := NewGraphConfigFromFile(b, debug, logf)
if err != nil {
return nil, err
}
if config == nil {
return nil, fmt.Errorf("%s: ParseConfigFromFile returned nil", Name)
return nil, fmt.Errorf("%s: NewGraphConfigFromFile returned nil", Name)
}
g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
@@ -169,7 +176,7 @@ func (obj *GAPI) Next() chan gapi.Next {
return
}
log.Printf("%s: Generating new graph...", Name)
obj.data.Logf("generating new graph...")
next := gapi.Next{
//Exit: true, // TODO: for permanent shutdown!
Err: err,

204
yamlgraph/gconfig.go
View File

@@ -19,13 +19,11 @@
package yamlgraph
import (
"errors"
"fmt"
"log"
"strings"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
errwrap "github.com/pkg/errors"
"gopkg.in/yaml.v2"
@@ -50,18 +48,6 @@ type Edge struct {
Notify bool `yaml:"notify"`
}
// ResourceData are the parameters for resource format.
type ResourceData struct {
Name string `yaml:"name"`
}
// Resource is the object that unmarshalls resources.
type Resource struct {
ResourceData
unmarshal func(interface{}) error
resource resources.Res
}
// Resources is the object that unmarshalls list of resources.
type Resources struct {
Resources map[string][]Resource `yaml:"resources"`
@@ -73,42 +59,19 @@ type GraphConfigData struct {
Collector []collectorResConfig `yaml:"collect"`
Edges []Edge `yaml:"edges"`
Comment string `yaml:"comment"`
Remote string `yaml:"remote"`
}
// GraphConfig is the data structure that describes a single graph to run.
type GraphConfig struct {
GraphConfigData
ResList []resources.Res
// ResourceData are the parameters for resource format.
type ResourceData struct {
Name string `yaml:"name"`
}
// UnmarshalYAML unmarshalls the complete graph.
func (c *GraphConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
// Unmarshal the graph data, except the resources
if err := unmarshal(&c.GraphConfigData); err != nil {
return err
}
// Resource is the object that unmarshalls resources.
type Resource struct {
ResourceData
// Unmarshal resources
var list Resources
list.Resources = map[string][]Resource{}
if err := unmarshal(&list); err != nil {
return err
}
// Finish unmarshalling by giving to each resource its kind
// and store each resource in the graph
for kind, resList := range list.Resources {
for _, res := range resList {
err := res.Decode(kind)
if err != nil {
return err
}
c.ResList = append(c.ResList, res.resource)
}
}
return nil
resource engine.Res
unmarshal func(interface{}) error
}
// UnmarshalYAML is the first stage for unmarshaling of resources.
@@ -120,11 +83,16 @@ func (r *Resource) UnmarshalYAML(unmarshal func(interface{}) error) error {
// Decode is the second stage for unmarshaling of resources (knowing their
// kind).
func (r *Resource) Decode(kind string) (err error) {
r.resource, err = resources.NewResource(kind)
if kind == "" {
return fmt.Errorf("can't set empty kind") // bug?
}
r.resource, err = engine.NewResource(kind)
if err != nil {
return err
}
// i think this erases the `SetKind` that happens with the NewResource
// so as a result, we need to do it again below... this is a hack...
err = r.unmarshal(r.resource)
if err != nil {
return err
@@ -132,25 +100,75 @@ func (r *Resource) Decode(kind string) (err error) {
// set resource name and kind
r.resource.SetName(r.Name)
r.resource.SetKind(strings.ToLower(kind)) // gets overwritten, so set it
// meta already gets unmarshalled properly with the correct defaults
r.resource.SetKind(kind)
// TODO: I don't think meta is getting unmarshalled properly anymore
return
}
// Parse parses a data stream into the graph structure.
func (c *GraphConfig) Parse(data []byte) error {
if err := yaml.Unmarshal(data, c); err != nil {
// GraphConfig is the data structure that describes a single graph to run.
type GraphConfig struct {
GraphConfigData
ResList []engine.Res
Debug bool
Logf func(format string, v ...interface{})
}
// NewGraphConfigFromFile takes data and returns the graph config structure.
func NewGraphConfigFromFile(data []byte, debug bool, logf func(format string, v ...interface{})) (*GraphConfig, error) {
var config GraphConfig
config.Debug = debug
config.Logf = logf
if err := config.Parse(data); err != nil {
return nil, errwrap.Wrapf(err, "parse error")
}
return &config, nil
}
// UnmarshalYAML unmarshalls the complete graph.
func (obj *GraphConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
// Unmarshal the graph data, except the resources
if err := unmarshal(&obj.GraphConfigData); err != nil {
return err
}
if c.Graph == "" {
return errors.New("graph config: invalid graph")
// unmarshal resources
var list Resources
list.Resources = map[string][]Resource{}
if err := unmarshal(&list); err != nil {
return err
}
// finish unmarshalling by giving to each resource its kind
// and store each resource in the graph
for kind, resList := range list.Resources {
for _, res := range resList {
err := res.Decode(kind)
if err != nil {
return err
}
obj.ResList = append(obj.ResList, res.resource)
}
}
return nil
}
// Parse parses a data stream into the graph structure.
func (obj *GraphConfig) Parse(data []byte) error {
if err := yaml.Unmarshal(data, obj); err != nil {
return err
}
if obj.Graph == "" {
return fmt.Errorf("invalid graph")
}
return nil
}
// NewGraphFromConfig transforms a GraphConfig struct into a new graph.
// FIXME: remove any possibly left over, now obsolete graph diff code from here!
func (c *GraphConfig) NewGraphFromConfig(hostname string, world resources.World, noop bool) (*pgraph.Graph, error) {
func (obj *GraphConfig) NewGraphFromConfig(hostname string, world engine.World, noop bool) (*pgraph.Graph, error) {
// hostname is the uuid for the host
var graph *pgraph.Graph // new graph to return
@@ -162,25 +180,30 @@ func (c *GraphConfig) NewGraphFromConfig(hostname string, world resources.World,
var lookup = make(map[string]map[string]pgraph.Vertex)
//log.Printf("%+v", config) // debug
// TODO: if defined (somehow)...
graph.SetName(c.Graph) // set graph name
graph.SetName(obj.Graph) // set graph name
var keep []pgraph.Vertex // list of vertex which are the same in new graph
var resourceList []resources.Res // list of resources to export
var keep []pgraph.Vertex // list of vertex which are the same in new graph
var resourceList []engine.Res // list of resources to export
// Resources
for _, res := range c.ResList {
kind := res.GetKind()
for _, res := range obj.ResList {
kind := res.Kind()
if kind == "" {
return nil, fmt.Errorf("resource has an empty kind") // bug?
}
if _, exists := lookup[kind]; !exists {
lookup[kind] = make(map[string]pgraph.Vertex)
}
// XXX: should we export based on a @@ prefix, or a metaparam
// like exported => true || exported => (host pattern)||(other pattern?)
if !strings.HasPrefix(res.GetName(), "@@") { // not exported resource
if !strings.HasPrefix(res.Name(), "@@") { // not exported resource
fn := func(v pgraph.Vertex) (bool, error) {
return resources.VtoR(v).Compare(res), nil
r, ok := v.(engine.Res)
if !ok {
return false, fmt.Errorf("not a Res")
}
return engine.ResCmp(r, res) == nil, nil
}
v, err := graph.VertexMatchFn(fn)
if err != nil {
@@ -190,12 +213,12 @@ func (c *GraphConfig) NewGraphFromConfig(hostname string, world resources.World,
v = res // a standalone res can be a vertex
graph.AddVertex(v) // call standalone in case not part of an edge
}
lookup[kind][res.GetName()] = v // used for constructing edges
keep = append(keep, v) // append
lookup[kind][res.Name()] = v // used for constructing edges
keep = append(keep, v) // append
} else if !noop { // do not export any resources if noop
// store for addition to backend storage...
res.SetName(res.GetName()[2:]) // slice off @@
res.SetName(res.Name()[2:]) // slice off @@
resourceList = append(resourceList, res)
}
}
@@ -208,7 +231,7 @@ func (c *GraphConfig) NewGraphFromConfig(hostname string, world resources.World,
// lookup from backend (usually etcd)
var hostnameFilter []string // empty to get from everyone
kindFilter := []string{}
for _, t := range c.Collector {
for _, t := range obj.Collector {
kind := strings.ToLower(t.Kind)
kindFilter = append(kindFilter, kind)
}
@@ -224,40 +247,48 @@ func (c *GraphConfig) NewGraphFromConfig(hostname string, world resources.World,
for _, res := range resourceList {
matched := false
// see if we find a collect pattern that matches
for _, t := range c.Collector {
for _, t := range obj.Collector {
kind := strings.ToLower(t.Kind)
// use t.Kind and optionally t.Pattern to collect from storage
log.Printf("Collect: %v; Pattern: %v", kind, t.Pattern)
obj.Logf("collect: %s; pattern: %v", kind, t.Pattern)
// XXX: expand to more complex pattern matching here...
if res.GetKind() != kind {
if res.Kind() != kind {
continue
}
if matched {
// we've already matched this resource, should we match again?
log.Printf("Config: Warning: Matching %s again!", res)
obj.Logf("warning: matching %s again!", res)
}
matched = true
// collect resources but add the noop metaparam
//if noop { // now done in mgmtmain
// res.Meta().Noop = noop
//if noop { // now done in main lib
// res.MetaParams().Noop = noop
//}
if t.Pattern != "" { // XXX: simplistic for now
res.CollectPattern(t.Pattern) // res.Dirname = t.Pattern
if xres, ok := res.(engine.CollectableRes); ok {
xres.CollectPattern(t.Pattern) // res.Dirname = t.Pattern
}
}
log.Printf("Collect: %s: collected!", res)
obj.Logf("collected: %s", res)
// XXX: similar to other resource add code:
if _, exists := lookup[kind]; !exists {
lookup[kind] = make(map[string]pgraph.Vertex)
}
// FIXME: do we need to expand this match function with
// the additional Cmp properties found in Meta, etc...?
fn := func(v pgraph.Vertex) (bool, error) {
return resources.VtoR(v).Compare(res), nil
r, ok := v.(engine.Res)
if !ok {
return false, fmt.Errorf("not a Res")
}
return engine.ResCmp(r, res) == nil, nil
}
v, err := graph.VertexMatchFn(fn)
if err != nil {
@@ -267,14 +298,14 @@ func (c *GraphConfig) NewGraphFromConfig(hostname string, world resources.World,
v = res // a standalone res can be a vertex
graph.AddVertex(v) // call standalone in case not part of an edge
}
lookup[kind][res.GetName()] = v // used for constructing edges
keep = append(keep, v) // append
lookup[kind][res.Name()] = v // used for constructing edges
keep = append(keep, v) // append
//break // let's see if another resource even matches
}
}
for _, e := range c.Edges {
for _, e := range obj.Edges {
if _, ok := lookup[strings.ToLower(e.From.Kind)]; !ok {
return nil, fmt.Errorf("can't find 'from' resource")
}
@@ -289,7 +320,7 @@ func (c *GraphConfig) NewGraphFromConfig(hostname string, world resources.World,
}
from := lookup[strings.ToLower(e.From.Kind)][e.From.Name]
to := lookup[strings.ToLower(e.To.Kind)][e.To.Name]
edge := &resources.Edge{
edge := &engine.Edge{
Name: e.Name,
Notify: e.Notify,
}
@@ -298,14 +329,3 @@ func (c *GraphConfig) NewGraphFromConfig(hostname string, world resources.World,
return graph, nil
}
// ParseConfigFromFile takes a filename and returns the graph config structure.
func ParseConfigFromFile(data []byte) *GraphConfig {
var config GraphConfig
if err := config.Parse(data); err != nil {
log.Printf("Config: Error: ParseConfigFromFile: Parse: %v", err)
return nil
}
return &config
}