diff --git a/lib/cli.go b/lib/cli.go index 5c0cb0e2..27bbf2dc 100644 --- a/lib/cli.go +++ b/lib/cli.go @@ -26,6 +26,7 @@ import ( "github.com/purpleidea/mgmt/puppet" "github.com/purpleidea/mgmt/yamlgraph" + "github.com/purpleidea/mgmt/yamlgraph2" "github.com/urfave/cli" ) @@ -71,6 +72,14 @@ func run(c *cli.Context) error { File: &y, } } + if y := c.String("yaml2"); c.IsSet("yaml2") { + if obj.GAPI != nil { + return fmt.Errorf("can't combine YAMLv2 GAPI with existing GAPI") + } + obj.GAPI = &yamlgraph2.GAPI{ + File: &y, + } + } if p := c.String("puppet"); c.IsSet("puppet") { if obj.GAPI != nil { return fmt.Errorf("can't combine puppet GAPI with existing GAPI") @@ -205,6 +214,11 @@ func CLI(program, version string, flags Flags) error { Value: "", Usage: "yaml graph definition to run", }, + cli.StringFlag{ + Name: "yaml2", + Value: "", + Usage: "yaml graph definition to run (parser v2)", + }, cli.StringFlag{ Name: "puppet, p", Value: "", diff --git a/resources/augeas.go b/resources/augeas.go index 6ac72139..42387975 100644 --- a/resources/augeas.go +++ b/resources/augeas.go @@ -40,6 +40,7 @@ const ( func init() { gob.Register(&AugeasRes{}) + RegisterResource("augeas", func() Res { return &AugeasRes{} }) } // AugeasRes is a resource that enables you to use the augeas resource. diff --git a/resources/exec.go b/resources/exec.go index 52fa6518..29fd4350 100644 --- a/resources/exec.go +++ b/resources/exec.go @@ -34,6 +34,7 @@ import ( func init() { gob.Register(&ExecRes{}) + RegisterResource("exec", func() Res { return &ExecRes{} }) } // ExecRes is an exec resource for running commands. diff --git a/resources/file.go b/resources/file.go index 6ec9989b..13725f45 100644 --- a/resources/file.go +++ b/resources/file.go @@ -42,6 +42,7 @@ import ( func init() { gob.Register(&FileRes{}) + RegisterResource("file", func() Res { return &FileRes{} }) } // FileRes is a file and directory resource. diff --git a/resources/hostname.go b/resources/hostname.go index 0a10532d..8872f92e 100644 --- a/resources/hostname.go +++ b/resources/hostname.go @@ -35,6 +35,7 @@ var ErrResourceInsufficientParameters = errors.New( "Insufficient parameters for this resource") func init() { + RegisterResource("hostname", func() Res { return &HostnameRes{} }) gob.Register(&HostnameRes{}) } diff --git a/resources/kv.go b/resources/kv.go index 8c2bca6f..c61723fd 100644 --- a/resources/kv.go +++ b/resources/kv.go @@ -27,6 +27,7 @@ import ( ) func init() { + RegisterResource("kv", func() Res { return &KVRes{} }) gob.Register(&KVRes{}) } diff --git a/resources/msg.go b/resources/msg.go index 458ea6a6..e76faaac 100644 --- a/resources/msg.go +++ b/resources/msg.go @@ -28,6 +28,7 @@ import ( ) func init() { + RegisterResource("msg", func() Res { return &MsgRes{} }) gob.Register(&MsgRes{}) } diff --git a/resources/noop.go b/resources/noop.go index 7b382a94..8268afc3 100644 --- a/resources/noop.go +++ b/resources/noop.go @@ -24,6 +24,7 @@ import ( ) func init() { + RegisterResource("noop", func() Res { return &NoopRes{} }) gob.Register(&NoopRes{}) } diff --git a/resources/nspawn.go b/resources/nspawn.go index 85788fbf..735ae70b 100644 --- a/resources/nspawn.go +++ b/resources/nspawn.go @@ -41,6 +41,7 @@ const ( ) func init() { + RegisterResource("nspawn", func() Res { return &NspawnRes{} }) gob.Register(&NspawnRes{}) } diff --git a/resources/password.go b/resources/password.go index 255b478d..69d3641b 100644 --- a/resources/password.go +++ b/resources/password.go @@ -34,6 +34,7 @@ import ( ) func init() { + RegisterResource("password", func() Res { return &PasswordRes{} }) gob.Register(&PasswordRes{}) } diff --git a/resources/pkg.go b/resources/pkg.go index 225ad1b5..2cc4f115 100644 --- a/resources/pkg.go +++ b/resources/pkg.go @@ -31,6 +31,7 @@ import ( ) func init() { + RegisterResource("pkg", func() Res { return &PkgRes{} }) gob.Register(&PkgRes{}) } diff --git a/resources/resources.go b/resources/resources.go index 93750b7a..50ef51e5 100644 --- a/resources/resources.go +++ b/resources/resources.go @@ -38,6 +38,24 @@ import ( "golang.org/x/time/rate" ) +var registeredResources = map[string]func() Res{} + +// RegisterResource registers a new resource by providing a constructor +// function that returns a resource object ready to be unmarshalled from YAML. +func RegisterResource(name string, creator func() Res) { + registeredResources[name] = creator +} + +// NewEmptyNamedResource returns an empty resource object from a registered +// type, ready to be unmarshalled. +func NewEmptyNamedResource(name string) (Res, error) { + fn, ok := registeredResources[name] + if !ok { + return nil, fmt.Errorf("no resource named %s available", name) + } + return fn(), nil +} + //go:generate stringer -type=ResState -output=resstate_stringer.go // The ResState type represents the current activity state of each resource. diff --git a/resources/svc.go b/resources/svc.go index 7041b0b4..7a71c6a0 100644 --- a/resources/svc.go +++ b/resources/svc.go @@ -33,6 +33,7 @@ import ( ) func init() { + RegisterResource("svc", func() Res { return &SvcRes{} }) gob.Register(&SvcRes{}) } diff --git a/resources/timer.go b/resources/timer.go index fd49d82d..a0875b57 100644 --- a/resources/timer.go +++ b/resources/timer.go @@ -25,6 +25,7 @@ import ( ) func init() { + RegisterResource("timer", func() Res { return &TimerRes{} }) gob.Register(&TimerRes{}) } diff --git a/resources/virt.go b/resources/virt.go index c2b97124..1b2bd1a1 100644 --- a/resources/virt.go +++ b/resources/virt.go @@ -37,6 +37,7 @@ import ( ) func init() { + RegisterResource("virt", func() Res { return &VirtRes{} }) gob.Register(&VirtRes{}) } diff --git a/yamlgraph2/gapi.go b/yamlgraph2/gapi.go new file mode 100644 index 00000000..bc71ebdf --- /dev/null +++ b/yamlgraph2/gapi.go @@ -0,0 +1,128 @@ +// Mgmt +// Copyright (C) 2013-2017+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package yamlgraph2 + +import ( + "fmt" + "log" + "sync" + + "github.com/purpleidea/mgmt/gapi" + "github.com/purpleidea/mgmt/pgraph" + "github.com/purpleidea/mgmt/recwatch" +) + +// GAPI implements the main yamlgraph GAPI interface. +type GAPI struct { + File *string // yaml graph definition to use; nil if undefined + + data gapi.Data + initialized bool + closeChan chan struct{} + wg sync.WaitGroup // sync group for tunnel go routines + configWatcher *recwatch.ConfigWatcher +} + +// NewGAPI creates a new yamlgraph GAPI struct and calls Init(). +func NewGAPI(data gapi.Data, file *string) (*GAPI, error) { + obj := &GAPI{ + File: file, + } + return obj, obj.Init(data) +} + +// Init initializes the yamlgraph GAPI struct. +func (obj *GAPI) Init(data gapi.Data) error { + if obj.initialized { + return fmt.Errorf("already initialized") + } + if obj.File == nil { + return fmt.Errorf("the File param must be specified") + } + obj.data = data // store for later + obj.closeChan = make(chan struct{}) + obj.initialized = true + obj.configWatcher = recwatch.NewConfigWatcher() + return nil +} + +// Graph returns a current Graph. +func (obj *GAPI) Graph() (*pgraph.Graph, error) { + if !obj.initialized { + return nil, fmt.Errorf("yamlgraph: GAPI is not initialized") + } + + config := ParseConfigFromFile(*obj.File) + if config == nil { + return nil, fmt.Errorf("yamlgraph: ParseConfigFromFile returned nil") + } + + g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop) + return g, err +} + +// Next returns nil errors every time there could be a new graph. +func (obj *GAPI) Next() chan error { + if obj.data.NoWatch { + return nil + } + ch := make(chan error) + obj.wg.Add(1) + go func() { + defer obj.wg.Done() + defer close(ch) // this will run before the obj.wg.Done() + if !obj.initialized { + ch <- fmt.Errorf("yamlgraph: GAPI is not initialized") + return + } + configChan := obj.configWatcher.ConfigWatch(*obj.File) // simple + for { + select { + case err, ok := <-configChan: // returns nil events on ok! + if !ok { // the channel closed! + return + } + log.Printf("yamlgraph: Generating new graph...") + select { + case ch <- err: // trigger a run (send a msg) + if err != nil { + return + } + // unblock if we exit while waiting to send! + case <-obj.closeChan: + return + } + case <-obj.closeChan: + return + } + } + }() + return ch +} + +// Close shuts down the yamlgraph GAPI. +func (obj *GAPI) Close() error { + if !obj.initialized { + return fmt.Errorf("yamlgraph: GAPI is not initialized") + } + obj.configWatcher.Close() + close(obj.closeChan) + obj.wg.Wait() + obj.initialized = false // closed = true + return nil +} diff --git a/yamlgraph2/gconfig.go b/yamlgraph2/gconfig.go new file mode 100644 index 00000000..eed08376 --- /dev/null +++ b/yamlgraph2/gconfig.go @@ -0,0 +1,301 @@ +// Mgmt +// Copyright (C) 2013-2017+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package yamlgraph2 provides the facilities for loading a graph from a yaml file. +package yamlgraph2 + +import ( + "errors" + "fmt" + "io/ioutil" + "log" + "strings" + + "github.com/purpleidea/mgmt/pgraph" + "github.com/purpleidea/mgmt/resources" + + "gopkg.in/yaml.v2" +) + +type collectorResConfig struct { + Kind string `yaml:"kind"` + Pattern string `yaml:"pattern"` // XXX: Not Implemented +} + +// Vertex is the data structure of a vertex. +type Vertex struct { + Kind string `yaml:"kind"` + Name string `yaml:"name"` +} + +// Edge is the data structure of an edge. +type Edge struct { + Name string `yaml:"name"` + From Vertex `yaml:"from"` + To Vertex `yaml:"to"` + Notify bool `yaml:"notify"` +} + +// ResourceData are the parameters for resource format. +type ResourceData struct { + Name string `yaml:"name"` + Meta resources.MetaParams `yaml:"meta"` +} + +// Resource is the object that unmarshalls resources. +type Resource struct { + ResourceData + unmarshal func(interface{}) error + resource resources.Res +} + +// Resources is the object that unmarshalls list of resources. +type Resources struct { + Resources map[string][]Resource `yaml:"resources"` +} + +// GraphConfigData contains the graph data for GraphConfig. +type GraphConfigData struct { + Graph string `yaml:"graph"` + Collector []collectorResConfig `yaml:"collect"` + Edges []Edge `yaml:"edges"` + Comment string `yaml:"comment"` + Remote string `yaml:"remote"` +} + +// GraphConfig is the data structure that describes a single graph to run. +type GraphConfig struct { + GraphConfigData + ResList []resources.Res +} + +// UnmarshalYAML unmarshalls the complete graph. +func (c *GraphConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Unmarshal the graph data, except the resources + if err := unmarshal(&c.GraphConfigData); err != nil { + return err + } + + // Unmarshal resources + var list Resources + list.Resources = map[string][]Resource{} + if err := unmarshal(&list); err != nil { + return err + } + + // Finish unmarshalling by giving to each resource its kind + // and store each resource in the graph + for kind, resList := range list.Resources { + for _, res := range resList { + err := res.Decode(kind) + if err != nil { + return err + } + c.ResList = append(c.ResList, res.resource) + } + } + + return nil +} + +// UnmarshalYAML is the first stage for unmarshaling of resources. +func (r *Resource) UnmarshalYAML(unmarshal func(interface{}) error) error { + r.unmarshal = unmarshal + return unmarshal(&r.ResourceData) +} + +// Decode is the second stage for unmarshaling of resources (knowing their +// kind). +func (r *Resource) Decode(kind string) (err error) { + r.resource, err = resources.NewEmptyNamedResource(kind) + if err != nil { + return err + } + + err = r.unmarshal(r.resource) + if err != nil { + return err + } + + // Set resource name, meta and kind + r.resource.SetName(r.Name) + r.resource.SetKind(strings.ToLower(kind)) + meta := r.resource.Meta() + *meta = r.Meta + return +} + +// Parse parses a data stream into the graph structure. +func (c *GraphConfig) Parse(data []byte) error { + if err := yaml.Unmarshal(data, c); err != nil { + return err + } + if c.Graph == "" { + return errors.New("graph config: invalid graph") + } + return nil +} + +// NewGraphFromConfig transforms a GraphConfig struct into a new graph. +// FIXME: remove any possibly left over, now obsolete graph diff code from here! +func (c *GraphConfig) NewGraphFromConfig(hostname string, world resources.World, noop bool) (*pgraph.Graph, error) { + // hostname is the uuid for the host + + var graph *pgraph.Graph // new graph to return + graph = pgraph.NewGraph("Graph") // give graph a default name + + var lookup = make(map[string]map[string]*pgraph.Vertex) + + //log.Printf("%+v", config) // debug + + // TODO: if defined (somehow)... + graph.SetName(c.Graph) // set graph name + + var keep []*pgraph.Vertex // list of vertex which are the same in new graph + var resourceList []resources.Res // list of resources to export + + // Resources + for _, res := range c.ResList { + kind := res.Kind() + if _, exists := lookup[kind]; !exists { + lookup[kind] = make(map[string]*pgraph.Vertex) + } + // XXX: should we export based on a @@ prefix, or a metaparam + // like exported => true || exported => (host pattern)||(other pattern?) + if !strings.HasPrefix(res.GetName(), "@@") { // not exported resource + v := graph.CompareMatch(res) + if v == nil { // no match found + v = pgraph.NewVertex(res) + graph.AddVertex(v) // call standalone in case not part of an edge + } + lookup[kind][res.GetName()] = v // used for constructing edges + keep = append(keep, v) // append + + } else if !noop { // do not export any resources if noop + // store for addition to backend storage... + res.SetName(res.GetName()[2:]) // slice off @@ + res.SetKind(kind) // cheap init + resourceList = append(resourceList, res) + } + } + + // store in backend (usually etcd) + if err := world.ResExport(resourceList); err != nil { + return nil, fmt.Errorf("Config: Could not export resources: %v", err) + } + + // lookup from backend (usually etcd) + var hostnameFilter []string // empty to get from everyone + kindFilter := []string{} + for _, t := range c.Collector { + kind := strings.ToLower(t.Kind) + kindFilter = append(kindFilter, kind) + } + // do all the graph look ups in one single step, so that if the backend + // database changes, we don't have a partial state of affairs... + if len(kindFilter) > 0 { // if kindFilter is empty, don't need to do lookups! + var err error + resourceList, err = world.ResCollect(hostnameFilter, kindFilter) + if err != nil { + return nil, fmt.Errorf("Config: Could not collect resources: %v", err) + } + } + for _, res := range resourceList { + matched := false + // see if we find a collect pattern that matches + for _, t := range c.Collector { + kind := strings.ToLower(t.Kind) + // use t.Kind and optionally t.Pattern to collect from storage + log.Printf("Collect: %v; Pattern: %v", kind, t.Pattern) + + // XXX: expand to more complex pattern matching here... + if res.Kind() != kind { + continue + } + + if matched { + // we've already matched this resource, should we match again? + log.Printf("Config: Warning: Matching %s[%s] again!", kind, res.GetName()) + } + matched = true + + // collect resources but add the noop metaparam + //if noop { // now done in mgmtmain + // res.Meta().Noop = noop + //} + + if t.Pattern != "" { // XXX: simplistic for now + res.CollectPattern(t.Pattern) // res.Dirname = t.Pattern + } + + log.Printf("Collect: %s[%s]: collected!", kind, res.GetName()) + + // XXX: similar to other resource add code: + if _, exists := lookup[kind]; !exists { + lookup[kind] = make(map[string]*pgraph.Vertex) + } + v := graph.CompareMatch(res) + if v == nil { // no match found + v = pgraph.NewVertex(res) + graph.AddVertex(v) // call standalone in case not part of an edge + } + lookup[kind][res.GetName()] = v // used for constructing edges + keep = append(keep, v) // append + + //break // let's see if another resource even matches + } + } + + for _, e := range c.Edges { + if _, ok := lookup[strings.ToLower(e.From.Kind)]; !ok { + return nil, fmt.Errorf("can't find 'from' resource") + } + if _, ok := lookup[strings.ToLower(e.To.Kind)]; !ok { + return nil, fmt.Errorf("can't find 'to' resource") + } + if _, ok := lookup[strings.ToLower(e.From.Kind)][e.From.Name]; !ok { + return nil, fmt.Errorf("can't find 'from' name") + } + if _, ok := lookup[strings.ToLower(e.To.Kind)][e.To.Name]; !ok { + return nil, fmt.Errorf("can't find 'to' name") + } + from := lookup[strings.ToLower(e.From.Kind)][e.From.Name] + to := lookup[strings.ToLower(e.To.Kind)][e.To.Name] + edge := pgraph.NewEdge(e.Name) + edge.Notify = e.Notify + graph.AddEdge(from, to, edge) + } + + return graph, nil +} + +// ParseConfigFromFile takes a filename and returns the graph config structure. +func ParseConfigFromFile(filename string) *GraphConfig { + data, err := ioutil.ReadFile(filename) + if err != nil { + log.Printf("Config: Error: ParseConfigFromFile: File: %v", err) + return nil + } + + var config GraphConfig + if err := config.Parse(data); err != nil { + log.Printf("Config: Error: ParseConfigFromFile: Parse: %v", err) + return nil + } + + return &config +}