lourenco/mgmt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lib, yamlgraph: Remove old yamlgraph GAPI frontend

Browse Source 
I should have removed this a long time ago, but didn't. Now it's done.
The new v2 frontend is loosing the v2 name and just replacing v1.
This commit is contained in:
James Shubin
2018-03-13 13:02:11 -04:00
parent 6e0dfdb16f
commit 4d8faeb826
4 changed files with 109 additions and 586 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
lib/deploy.go
View File

@@ -30,7 +30,6 @@ import (
_ "github.com/purpleidea/mgmt/lang"
_ "github.com/purpleidea/mgmt/puppet"
_ "github.com/purpleidea/mgmt/yamlgraph"
_ "github.com/purpleidea/mgmt/yamlgraph2"
"github.com/google/uuid"
errwrap "github.com/pkg/errors"

182
yamlgraph/gconfig.go
View File

@@ -22,7 +22,6 @@ import (
"errors"
"fmt"
"log"
"reflect"
"strings"
"github.com/purpleidea/mgmt/pgraph"
@@ -51,41 +50,93 @@ type Edge struct {
Notify bool `yaml:"notify"`
}
// Resources is the data structure of the set of resources.
type Resources struct {
// in alphabetical order
Augeas []*resources.AugeasRes `yaml:"augeas"`
AwsEc2 []*resources.AwsEc2Res `yaml:"aws:ec2"`
Exec []*resources.ExecRes `yaml:"exec"`
File []*resources.FileRes `yaml:"file"`
Graph []*resources.GraphRes `yaml:"graph"`
Group []*resources.GroupRes `yaml:"group"`
Hostname []*resources.HostnameRes `yaml:"hostname"`
KV []*resources.KVRes `yaml:"kv"`
Msg []*resources.MsgRes `yaml:"msg"`
Net []*resources.NetRes `yaml:"net"`
Noop []*resources.NoopRes `yaml:"noop"`
Nspawn []*resources.NspawnRes `yaml:"nspawn"`
Password []*resources.PasswordRes `yaml:"password"`
Pkg []*resources.PkgRes `yaml:"pkg"`
Print []*resources.PrintRes `yaml:"print"`
Svc []*resources.SvcRes `yaml:"svc"`
Test []*resources.TestRes `yaml:"test"`
Timer []*resources.TimerRes `yaml:"timer"`
User []*resources.UserRes `yaml:"user"`
Virt []*resources.VirtRes `yaml:"virt"`
// ResourceData are the parameters for resource format.
type ResourceData struct {
Name string `yaml:"name"`
}
// GraphConfig is the data structure that describes a single graph to run.
type GraphConfig struct {
// Resource is the object that unmarshalls resources.
type Resource struct {
ResourceData
unmarshal func(interface{}) error
resource resources.Res
}
// Resources is the object that unmarshalls list of resources.
type Resources struct {
Resources map[string][]Resource `yaml:"resources"`
}
// GraphConfigData contains the graph data for GraphConfig.
type GraphConfigData struct {
Graph string `yaml:"graph"`
Resources Resources `yaml:"resources"`
Collector []collectorResConfig `yaml:"collect"`
Edges []Edge `yaml:"edges"`
Comment string `yaml:"comment"`
Remote string `yaml:"remote"`
}
// GraphConfig is the data structure that describes a single graph to run.
type GraphConfig struct {
GraphConfigData
ResList []resources.Res
}
// UnmarshalYAML unmarshalls the complete graph.
func (c *GraphConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
// Unmarshal the graph data, except the resources
if err := unmarshal(&c.GraphConfigData); err != nil {
return err
}
// Unmarshal resources
var list Resources
list.Resources = map[string][]Resource{}
if err := unmarshal(&list); err != nil {
return err
}
// Finish unmarshalling by giving to each resource its kind
// and store each resource in the graph
for kind, resList := range list.Resources {
for _, res := range resList {
err := res.Decode(kind)
if err != nil {
return err
}
c.ResList = append(c.ResList, res.resource)
}
}
return nil
}
// UnmarshalYAML is the first stage for unmarshaling of resources.
func (r *Resource) UnmarshalYAML(unmarshal func(interface{}) error) error {
r.unmarshal = unmarshal
return unmarshal(&r.ResourceData)
}
// Decode is the second stage for unmarshaling of resources (knowing their
// kind).
func (r *Resource) Decode(kind string) (err error) {
r.resource, err = resources.NewResource(kind)
if err != nil {
return err
}
err = r.unmarshal(r.resource)
if err != nil {
return err
}
// set resource name and kind
r.resource.SetName(r.Name)
r.resource.SetKind(strings.ToLower(kind)) // gets overwritten, so set it
// meta already gets unmarshalled properly with the correct defaults
return
}
// Parse parses a data stream into the graph structure.
func (c *GraphConfig) Parse(data []byte) error {
if err := yaml.Unmarshal(data, c); err != nil {
@@ -118,52 +169,37 @@ func (c *GraphConfig) NewGraphFromConfig(hostname string, world resources.World,
var keep []pgraph.Vertex // list of vertex which are the same in new graph
var resourceList []resources.Res // list of resources to export
// use reflection to avoid duplicating code... better options welcome!
value := reflect.Indirect(reflect.ValueOf(c.Resources))
vtype := value.Type()
for i := 0; i < vtype.NumField(); i++ { // number of fields in struct
name := vtype.Field(i).Name // string of field name
field := value.FieldByName(name)
iface := field.Interface() // interface type of value
slice := reflect.ValueOf(iface)
kind := strings.ToLower(name)
for j := 0; j < slice.Len(); j++ { // loop through resources of same kind
x := slice.Index(j).Interface()
res, ok := x.(resources.Res) // convert to Res type
if !ok {
return nil, fmt.Errorf("Config: Error: Can't convert: %v of type: %T to Res", x, x)
}
res.SetKind(kind) // cheap init
//if noop { // now done in mgmtmain
// res.Meta().Noop = noop
//}
if _, exists := lookup[kind]; !exists {
lookup[kind] = make(map[string]pgraph.Vertex)
}
// XXX: should we export based on a @@ prefix, or a metaparam
// like exported => true || exported => (host pattern)||(other pattern?)
if !strings.HasPrefix(res.GetName(), "@@") { // not exported resource
fn := func(v pgraph.Vertex) (bool, error) {
return resources.VtoR(v).Compare(res), nil
}
v, err := graph.VertexMatchFn(fn)
if err != nil {
return nil, errwrap.Wrapf(err, "could not VertexMatchFn() resource")
}
if v == nil { // no match found
v = res // a standalone res can be a vertex
graph.AddVertex(v) // call standalone in case not part of an edge
}
lookup[kind][res.GetName()] = v // used for constructing edges
keep = append(keep, v) // append
} else if !noop { // do not export any resources if noop
// store for addition to backend storage...
res.SetName(res.GetName()[2:]) //slice off @@
resourceList = append(resourceList, res)
// Resources
for _, res := range c.ResList {
kind := res.GetKind()
if _, exists := lookup[kind]; !exists {
lookup[kind] = make(map[string]pgraph.Vertex)
}
// XXX: should we export based on a @@ prefix, or a metaparam
// like exported => true || exported => (host pattern)||(other pattern?)
if !strings.HasPrefix(res.GetName(), "@@") { // not exported resource
fn := func(v pgraph.Vertex) (bool, error) {
return resources.VtoR(v).Compare(res), nil
}
v, err := graph.VertexMatchFn(fn)
if err != nil {
return nil, errwrap.Wrapf(err, "could not VertexMatchFn() resource")
}
if v == nil { // no match found
v = res // a standalone res can be a vertex
graph.AddVertex(v) // call standalone in case not part of an edge
}
lookup[kind][res.GetName()] = v // used for constructing edges
keep = append(keep, v) // append
} else if !noop { // do not export any resources if noop
// store for addition to backend storage...
res.SetName(res.GetName()[2:]) // slice off @@
resourceList = append(resourceList, res)
}
}
// store in backend (usually etcd)
if err := world.ResExport(resourceList); err != nil {
return nil, fmt.Errorf("Config: Could not export resources: %v", err)
@@ -240,16 +276,16 @@ func (c *GraphConfig) NewGraphFromConfig(hostname string, world resources.World,
for _, e := range c.Edges {
if _, ok := lookup[strings.ToLower(e.From.Kind)]; !ok {
return nil, fmt.Errorf("can't find 'from' kind: %s", e.From.Kind)
return nil, fmt.Errorf("can't find 'from' resource")
}
if _, ok := lookup[strings.ToLower(e.To.Kind)]; !ok {
return nil, fmt.Errorf("can't find 'to' kind: %s", e.To.Kind)
return nil, fmt.Errorf("can't find 'to' resource")
}
if _, ok := lookup[strings.ToLower(e.From.Kind)][e.From.Name]; !ok {
return nil, fmt.Errorf("can't find 'from' name: %s", e.From.Name)
return nil, fmt.Errorf("can't find 'from' name")
}
if _, ok := lookup[strings.ToLower(e.To.Kind)][e.To.Name]; !ok {
return nil, fmt.Errorf("can't find 'to' name: %s", e.To.Name)
return nil, fmt.Errorf("can't find 'to' name")
}
from := lookup[strings.ToLower(e.From.Kind)][e.From.Name]
to := lookup[strings.ToLower(e.To.Kind)][e.To.Name]

201
yamlgraph2/gapi.go
View File

@@ -1,201 +0,0 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package yamlgraph2
import (
"fmt"
"log"
"sync"
"github.com/purpleidea/mgmt/gapi"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
errwrap "github.com/pkg/errors"
"github.com/urfave/cli"
)
const (
// Name is the name of this frontend.
Name = "yaml2"
// Start is the entry point filename that we use. It is arbitrary.
Start = "/start.yaml"
)
func init() {
gapi.Register(Name, func() gapi.GAPI { return &GAPI{} }) // register
}
// GAPI implements the main yamlgraph GAPI interface.
type GAPI struct {
InputURI string // input URI of file system containing yaml graph to use
data gapi.Data
initialized bool
closeChan chan struct{}
wg sync.WaitGroup // sync group for tunnel go routines
}
// Cli takes a cli.Context, and returns our GAPI if activated. All arguments
// should take the prefix of the registered name. On activation, if there are
// any validation problems, you should return an error. If this was not
// activated, then you should return a nil GAPI and a nil error.
func (obj *GAPI) Cli(c *cli.Context, fs resources.Fs) (*gapi.Deploy, error) {
if s := c.String(Name); c.IsSet(Name) {
if s == "" {
return nil, fmt.Errorf("input yaml is empty")
}
// single file input only
if err := gapi.CopyFileToFs(fs, s, Start); err != nil {
return nil, errwrap.Wrapf(err, "can't copy yaml from `%s` to `%s`", s, Start)
}
return &gapi.Deploy{
Name: Name,
Noop: c.GlobalBool("noop"),
Sema: c.GlobalInt("sema"),
GAPI: &GAPI{
InputURI: fs.URI(),
// TODO: add properties here...
},
}, nil
}
return nil, nil // we weren't activated!
}
// CliFlags returns a list of flags used by this deploy subcommand.
func (obj *GAPI) CliFlags() []cli.Flag {
return []cli.Flag{
cli.StringFlag{
Name: Name,
Value: "",
Usage: "yaml graph definition to run (parser v2)",
},
}
}
// Init initializes the yamlgraph GAPI struct.
func (obj *GAPI) Init(data gapi.Data) error {
if obj.initialized {
return fmt.Errorf("already initialized")
}
if obj.InputURI == "" {
return fmt.Errorf("the InputURI param must be specified")
}
obj.data = data // store for later
obj.closeChan = make(chan struct{})
obj.initialized = true
return nil
}
// Graph returns a current Graph.
func (obj *GAPI) Graph() (*pgraph.Graph, error) {
if !obj.initialized {
return nil, fmt.Errorf("%s: GAPI is not initialized", Name)
}
fs, err := obj.data.World.Fs(obj.InputURI) // open the remote file system
if err != nil {
return nil, errwrap.Wrapf(err, "can't load yaml from file system `%s`", obj.InputURI)
}
b, err := fs.ReadFile(Start) // read the single file out of it
if err != nil {
return nil, errwrap.Wrapf(err, "can't read yaml from file `%s`", Start)
}
config := ParseConfigFromFile(b)
if config == nil {
return nil, fmt.Errorf("%s: ParseConfigFromFile returned nil", Name)
}
g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
return g, err
}
// Next returns nil errors every time there could be a new graph.
func (obj *GAPI) Next() chan gapi.Next {
ch := make(chan gapi.Next)
obj.wg.Add(1)
go func() {
defer obj.wg.Done()
defer close(ch) // this will run before the obj.wg.Done()
if !obj.initialized {
next := gapi.Next{
Err: fmt.Errorf("%s: GAPI is not initialized", Name),
Exit: true, // exit, b/c programming error?
}
ch <- next
return
}
startChan := make(chan struct{}) // start signal
close(startChan) // kick it off!
var watchChan chan error
if obj.data.NoStreamWatch {
watchChan = nil
} else {
watchChan = obj.data.World.ResWatch()
}
for {
var err error
var ok bool
select {
case <-startChan: // kick the loop once at start
startChan = nil // disable
// pass
case err, ok = <-watchChan:
if !ok {
return
}
case <-obj.closeChan:
return
}
log.Printf("%s: Generating new graph...", Name)
next := gapi.Next{
//Exit: true, // TODO: for permanent shutdown!
Err: err,
}
select {
case ch <- next: // trigger a run (send a msg)
// TODO: if the error is really bad, we could:
//if err != nil {
// return
//}
// unblock if we exit while waiting to send!
case <-obj.closeChan:
return
}
}
}()
return ch
}
// Close shuts down the yamlgraph GAPI.
func (obj *GAPI) Close() error {
if !obj.initialized {
return fmt.Errorf("%s: GAPI is not initialized", Name)
}
close(obj.closeChan)
obj.wg.Wait()
obj.initialized = false // closed = true
return nil
}

311
yamlgraph2/gconfig.go
View File

@@ -1,311 +0,0 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package yamlgraph2 provides the facilities for loading a graph from a yaml file.
package yamlgraph2
import (
"errors"
"fmt"
"log"
"strings"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
errwrap "github.com/pkg/errors"
"gopkg.in/yaml.v2"
)
type collectorResConfig struct {
Kind string `yaml:"kind"`
Pattern string `yaml:"pattern"` // XXX: not implemented
}
// Vertex is the data structure of a vertex.
type Vertex struct {
Kind string `yaml:"kind"`
Name string `yaml:"name"`
}
// Edge is the data structure of an edge.
type Edge struct {
Name string `yaml:"name"`
From Vertex `yaml:"from"`
To Vertex `yaml:"to"`
Notify bool `yaml:"notify"`
}
// ResourceData are the parameters for resource format.
type ResourceData struct {
Name string `yaml:"name"`
}
// Resource is the object that unmarshalls resources.
type Resource struct {
ResourceData
unmarshal func(interface{}) error
resource resources.Res
}
// Resources is the object that unmarshalls list of resources.
type Resources struct {
Resources map[string][]Resource `yaml:"resources"`
}
// GraphConfigData contains the graph data for GraphConfig.
type GraphConfigData struct {
Graph string `yaml:"graph"`
Collector []collectorResConfig `yaml:"collect"`
Edges []Edge `yaml:"edges"`
Comment string `yaml:"comment"`
Remote string `yaml:"remote"`
}
// GraphConfig is the data structure that describes a single graph to run.
type GraphConfig struct {
GraphConfigData
ResList []resources.Res
}
// UnmarshalYAML unmarshalls the complete graph.
func (c *GraphConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
// Unmarshal the graph data, except the resources
if err := unmarshal(&c.GraphConfigData); err != nil {
return err
}
// Unmarshal resources
var list Resources
list.Resources = map[string][]Resource{}
if err := unmarshal(&list); err != nil {
return err
}
// Finish unmarshalling by giving to each resource its kind
// and store each resource in the graph
for kind, resList := range list.Resources {
for _, res := range resList {
err := res.Decode(kind)
if err != nil {
return err
}
c.ResList = append(c.ResList, res.resource)
}
}
return nil
}
// UnmarshalYAML is the first stage for unmarshaling of resources.
func (r *Resource) UnmarshalYAML(unmarshal func(interface{}) error) error {
r.unmarshal = unmarshal
return unmarshal(&r.ResourceData)
}
// Decode is the second stage for unmarshaling of resources (knowing their
// kind).
func (r *Resource) Decode(kind string) (err error) {
r.resource, err = resources.NewResource(kind)
if err != nil {
return err
}
err = r.unmarshal(r.resource)
if err != nil {
return err
}
// set resource name and kind
r.resource.SetName(r.Name)
r.resource.SetKind(strings.ToLower(kind)) // gets overwritten, so set it
// meta already gets unmarshalled properly with the correct defaults
return
}
// Parse parses a data stream into the graph structure.
func (c *GraphConfig) Parse(data []byte) error {
if err := yaml.Unmarshal(data, c); err != nil {
return err
}
if c.Graph == "" {
return errors.New("graph config: invalid graph")
}
return nil
}
// NewGraphFromConfig transforms a GraphConfig struct into a new graph.
// FIXME: remove any possibly left over, now obsolete graph diff code from here!
func (c *GraphConfig) NewGraphFromConfig(hostname string, world resources.World, noop bool) (*pgraph.Graph, error) {
// hostname is the uuid for the host
var graph *pgraph.Graph // new graph to return
var err error
graph, err = pgraph.NewGraph("Graph") // give graph a default name
if err != nil {
return nil, errwrap.Wrapf(err, "could not run NewGraphFromConfig() properly")
}
var lookup = make(map[string]map[string]pgraph.Vertex)
//log.Printf("%+v", config) // debug
// TODO: if defined (somehow)...
graph.SetName(c.Graph) // set graph name
var keep []pgraph.Vertex // list of vertex which are the same in new graph
var resourceList []resources.Res // list of resources to export
// Resources
for _, res := range c.ResList {
kind := res.GetKind()
if _, exists := lookup[kind]; !exists {
lookup[kind] = make(map[string]pgraph.Vertex)
}
// XXX: should we export based on a @@ prefix, or a metaparam
// like exported => true || exported => (host pattern)||(other pattern?)
if !strings.HasPrefix(res.GetName(), "@@") { // not exported resource
fn := func(v pgraph.Vertex) (bool, error) {
return resources.VtoR(v).Compare(res), nil
}
v, err := graph.VertexMatchFn(fn)
if err != nil {
return nil, errwrap.Wrapf(err, "could not VertexMatchFn() resource")
}
if v == nil { // no match found
v = res // a standalone res can be a vertex
graph.AddVertex(v) // call standalone in case not part of an edge
}
lookup[kind][res.GetName()] = v // used for constructing edges
keep = append(keep, v) // append
} else if !noop { // do not export any resources if noop
// store for addition to backend storage...
res.SetName(res.GetName()[2:]) // slice off @@
resourceList = append(resourceList, res)
}
}
// store in backend (usually etcd)
if err := world.ResExport(resourceList); err != nil {
return nil, fmt.Errorf("Config: Could not export resources: %v", err)
}
// lookup from backend (usually etcd)
var hostnameFilter []string // empty to get from everyone
kindFilter := []string{}
for _, t := range c.Collector {
kind := strings.ToLower(t.Kind)
kindFilter = append(kindFilter, kind)
}
// do all the graph look ups in one single step, so that if the backend
// database changes, we don't have a partial state of affairs...
if len(kindFilter) > 0 { // if kindFilter is empty, don't need to do lookups!
var err error
resourceList, err = world.ResCollect(hostnameFilter, kindFilter)
if err != nil {
return nil, fmt.Errorf("Config: Could not collect resources: %v", err)
}
}
for _, res := range resourceList {
matched := false
// see if we find a collect pattern that matches
for _, t := range c.Collector {
kind := strings.ToLower(t.Kind)
// use t.Kind and optionally t.Pattern to collect from storage
log.Printf("Collect: %v; Pattern: %v", kind, t.Pattern)
// XXX: expand to more complex pattern matching here...
if res.GetKind() != kind {
continue
}
if matched {
// we've already matched this resource, should we match again?
log.Printf("Config: Warning: Matching %s again!", res)
}
matched = true
// collect resources but add the noop metaparam
//if noop { // now done in mgmtmain
// res.Meta().Noop = noop
//}
if t.Pattern != "" { // XXX: simplistic for now
res.CollectPattern(t.Pattern) // res.Dirname = t.Pattern
}
log.Printf("Collect: %s: collected!", res)
// XXX: similar to other resource add code:
if _, exists := lookup[kind]; !exists {
lookup[kind] = make(map[string]pgraph.Vertex)
}
fn := func(v pgraph.Vertex) (bool, error) {
return resources.VtoR(v).Compare(res), nil
}
v, err := graph.VertexMatchFn(fn)
if err != nil {
return nil, errwrap.Wrapf(err, "could not VertexMatchFn() resource")
}
if v == nil { // no match found
v = res // a standalone res can be a vertex
graph.AddVertex(v) // call standalone in case not part of an edge
}
lookup[kind][res.GetName()] = v // used for constructing edges
keep = append(keep, v) // append
//break // let's see if another resource even matches
}
}
for _, e := range c.Edges {
if _, ok := lookup[strings.ToLower(e.From.Kind)]; !ok {
return nil, fmt.Errorf("can't find 'from' resource")
}
if _, ok := lookup[strings.ToLower(e.To.Kind)]; !ok {
return nil, fmt.Errorf("can't find 'to' resource")
}
if _, ok := lookup[strings.ToLower(e.From.Kind)][e.From.Name]; !ok {
return nil, fmt.Errorf("can't find 'from' name")
}
if _, ok := lookup[strings.ToLower(e.To.Kind)][e.To.Name]; !ok {
return nil, fmt.Errorf("can't find 'to' name")
}
from := lookup[strings.ToLower(e.From.Kind)][e.From.Name]
to := lookup[strings.ToLower(e.To.Kind)][e.To.Name]
edge := &resources.Edge{
Name: e.Name,
Notify: e.Notify,
}
graph.AddEdge(from, to, edge)
}
return graph, nil
}
// ParseConfigFromFile takes a filename and returns the graph config structure.
func ParseConfigFromFile(data []byte) *GraphConfig {
var config GraphConfig
if err := config.Parse(data); err != nil {
log.Printf("Config: Error: ParseConfigFromFile: Parse: %v", err)
return nil
}
return &config
}