yamlgraph: Refactor parsing for dynamic resource registration

Avoid use of the reflect package, and use an extensible list of registred
resource kinds. This also has the benefit of removing the empty VirtRes and
AugeasRes struct types when compiling without libvirt and libaugeas.

lib/cli.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/purpleidea/mgmt/puppet"
 	"github.com/purpleidea/mgmt/yamlgraph"
+	"github.com/purpleidea/mgmt/yamlgraph2"
 
 	"github.com/urfave/cli"
 )
@@ -71,6 +72,14 @@ func run(c *cli.Context) error {
 			File: &y,
 		}
 	}
+	if y := c.String("yaml2"); c.IsSet("yaml2") {
+		if obj.GAPI != nil {
+			return fmt.Errorf("can't combine YAMLv2 GAPI with existing GAPI")
+		}
+		obj.GAPI = &yamlgraph2.GAPI{
+			File: &y,
+		}
+	}
 	if p := c.String("puppet"); c.IsSet("puppet") {
 		if obj.GAPI != nil {
 			return fmt.Errorf("can't combine puppet GAPI with existing GAPI")
@@ -205,6 +214,11 @@ func CLI(program, version string, flags Flags) error {
 					Value: "",
 					Usage: "yaml graph definition to run",
 				},
+				cli.StringFlag{
+					Name:  "yaml2",
+					Value: "",
+					Usage: "yaml graph definition to run (parser v2)",
+				},
 				cli.StringFlag{
 					Name:  "puppet, p",
 					Value: "",

resources/augeas.go

@@ -40,6 +40,7 @@ const (
 
 func init() {
 	gob.Register(&AugeasRes{})
+	RegisterResource("augeas", func() Res { return &AugeasRes{} })
 }
 
 // AugeasRes is a resource that enables you to use the augeas resource.

resources/exec.go

@@ -34,6 +34,7 @@ import (
 
 func init() {
 	gob.Register(&ExecRes{})
+	RegisterResource("exec", func() Res { return &ExecRes{} })
 }
 
 // ExecRes is an exec resource for running commands.

resources/file.go

@@ -42,6 +42,7 @@ import (
 
 func init() {
 	gob.Register(&FileRes{})
+	RegisterResource("file", func() Res { return &FileRes{} })
 }
 
 // FileRes is a file and directory resource.

resources/hostname.go

@@ -35,6 +35,7 @@ var ErrResourceInsufficientParameters = errors.New(
 	"Insufficient parameters for this resource")
 
 func init() {
+	RegisterResource("hostname", func() Res { return &HostnameRes{} })
 	gob.Register(&HostnameRes{})
 }
 

resources/kv.go

@@ -27,6 +27,7 @@ import (
 )
 
 func init() {
+	RegisterResource("kv", func() Res { return &KVRes{} })
 	gob.Register(&KVRes{})
 }
 

resources/msg.go

@@ -28,6 +28,7 @@ import (
 )
 
 func init() {
+	RegisterResource("msg", func() Res { return &MsgRes{} })
 	gob.Register(&MsgRes{})
 }
 

resources/noop.go

@@ -24,6 +24,7 @@ import (
 )
 
 func init() {
+	RegisterResource("noop", func() Res { return &NoopRes{} })
 	gob.Register(&NoopRes{})
 }
 

resources/nspawn.go

@@ -41,6 +41,7 @@ const (
 )
 
 func init() {
+	RegisterResource("nspawn", func() Res { return &NspawnRes{} })
 	gob.Register(&NspawnRes{})
 }
 

resources/password.go

@@ -34,6 +34,7 @@ import (
 )
 
 func init() {
+	RegisterResource("password", func() Res { return &PasswordRes{} })
 	gob.Register(&PasswordRes{})
 }
 

resources/pkg.go

@@ -31,6 +31,7 @@ import (
 )
 
 func init() {
+	RegisterResource("pkg", func() Res { return &PkgRes{} })
 	gob.Register(&PkgRes{})
 }
 

resources/resources.go

@@ -38,6 +38,24 @@ import (
 	"golang.org/x/time/rate"
 )
 
+var registeredResources = map[string]func() Res{}
+
+// RegisterResource registers a new resource by providing a constructor
+// function that returns a resource object ready to be unmarshalled from YAML.
+func RegisterResource(name string, creator func() Res) {
+	registeredResources[name] = creator
+}
+
+// NewEmptyNamedResource returns an empty resource object from a registered
+// type, ready to be unmarshalled.
+func NewEmptyNamedResource(name string) (Res, error) {
+	fn, ok := registeredResources[name]
+	if !ok {
+		return nil, fmt.Errorf("no resource named %s available", name)
+	}
+	return fn(), nil
+}
+
 //go:generate stringer -type=ResState -output=resstate_stringer.go
 
 // The ResState type represents the current activity state of each resource.

resources/svc.go

@@ -33,6 +33,7 @@ import (
 )
 
 func init() {
+	RegisterResource("svc", func() Res { return &SvcRes{} })
 	gob.Register(&SvcRes{})
 }
 

resources/timer.go

@@ -25,6 +25,7 @@ import (
 )
 
 func init() {
+	RegisterResource("timer", func() Res { return &TimerRes{} })
 	gob.Register(&TimerRes{})
 }
 

resources/virt.go

@@ -37,6 +37,7 @@ import (
 )
 
 func init() {
+	RegisterResource("virt", func() Res { return &VirtRes{} })
 	gob.Register(&VirtRes{})
 }
 

yamlgraph2/gapi.go

@@ -0,0 +1,128 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package yamlgraph2
+
+import (
+	"fmt"
+	"log"
+	"sync"
+
+	"github.com/purpleidea/mgmt/gapi"
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/recwatch"
+)
+
+// GAPI implements the main yamlgraph GAPI interface.
+type GAPI struct {
+	File *string // yaml graph definition to use; nil if undefined
+
+	data          gapi.Data
+	initialized   bool
+	closeChan     chan struct{}
+	wg            sync.WaitGroup // sync group for tunnel go routines
+	configWatcher *recwatch.ConfigWatcher
+}
+
+// NewGAPI creates a new yamlgraph GAPI struct and calls Init().
+func NewGAPI(data gapi.Data, file *string) (*GAPI, error) {
+	obj := &GAPI{
+		File: file,
+	}
+	return obj, obj.Init(data)
+}
+
+// Init initializes the yamlgraph GAPI struct.
+func (obj *GAPI) Init(data gapi.Data) error {
+	if obj.initialized {
+		return fmt.Errorf("already initialized")
+	}
+	if obj.File == nil {
+		return fmt.Errorf("the File param must be specified")
+	}
+	obj.data = data // store for later
+	obj.closeChan = make(chan struct{})
+	obj.initialized = true
+	obj.configWatcher = recwatch.NewConfigWatcher()
+	return nil
+}
+
+// Graph returns a current Graph.
+func (obj *GAPI) Graph() (*pgraph.Graph, error) {
+	if !obj.initialized {
+		return nil, fmt.Errorf("yamlgraph: GAPI is not initialized")
+	}
+
+	config := ParseConfigFromFile(*obj.File)
+	if config == nil {
+		return nil, fmt.Errorf("yamlgraph: ParseConfigFromFile returned nil")
+	}
+
+	g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
+	return g, err
+}
+
+// Next returns nil errors every time there could be a new graph.
+func (obj *GAPI) Next() chan error {
+	if obj.data.NoWatch {
+		return nil
+	}
+	ch := make(chan error)
+	obj.wg.Add(1)
+	go func() {
+		defer obj.wg.Done()
+		defer close(ch) // this will run before the obj.wg.Done()
+		if !obj.initialized {
+			ch <- fmt.Errorf("yamlgraph: GAPI is not initialized")
+			return
+		}
+		configChan := obj.configWatcher.ConfigWatch(*obj.File) // simple
+		for {
+			select {
+			case err, ok := <-configChan: // returns nil events on ok!
+				if !ok { // the channel closed!
+					return
+				}
+				log.Printf("yamlgraph: Generating new graph...")
+				select {
+				case ch <- err: // trigger a run (send a msg)
+					if err != nil {
+						return
+					}
+				// unblock if we exit while waiting to send!
+				case <-obj.closeChan:
+					return
+				}
+			case <-obj.closeChan:
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+// Close shuts down the yamlgraph GAPI.
+func (obj *GAPI) Close() error {
+	if !obj.initialized {
+		return fmt.Errorf("yamlgraph: GAPI is not initialized")
+	}
+	obj.configWatcher.Close()
+	close(obj.closeChan)
+	obj.wg.Wait()
+	obj.initialized = false // closed = true
+	return nil
+}

yamlgraph2/gconfig.go

@@ -0,0 +1,301 @@
+// Mgmt
+// Copyright (C) 2013-2017+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package yamlgraph2 provides the facilities for loading a graph from a yaml file.
+package yamlgraph2
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"strings"
+
+	"github.com/purpleidea/mgmt/pgraph"
+	"github.com/purpleidea/mgmt/resources"
+
+	"gopkg.in/yaml.v2"
+)
+
+type collectorResConfig struct {
+	Kind    string `yaml:"kind"`
+	Pattern string `yaml:"pattern"` // XXX: Not Implemented
+}
+
+// Vertex is the data structure of a vertex.
+type Vertex struct {
+	Kind string `yaml:"kind"`
+	Name string `yaml:"name"`
+}
+
+// Edge is the data structure of an edge.
+type Edge struct {
+	Name   string `yaml:"name"`
+	From   Vertex `yaml:"from"`
+	To     Vertex `yaml:"to"`
+	Notify bool   `yaml:"notify"`
+}
+
+// ResourceData are the parameters for resource format.
+type ResourceData struct {
+	Name string               `yaml:"name"`
+	Meta resources.MetaParams `yaml:"meta"`
+}
+
+// Resource is the object that unmarshalls resources.
+type Resource struct {
+	ResourceData
+	unmarshal func(interface{}) error
+	resource  resources.Res
+}
+
+// Resources is the object that unmarshalls list of resources.
+type Resources struct {
+	Resources map[string][]Resource `yaml:"resources"`
+}
+
+// GraphConfigData contains the graph data for GraphConfig.
+type GraphConfigData struct {
+	Graph     string               `yaml:"graph"`
+	Collector []collectorResConfig `yaml:"collect"`
+	Edges     []Edge               `yaml:"edges"`
+	Comment   string               `yaml:"comment"`
+	Remote    string               `yaml:"remote"`
+}
+
+// GraphConfig is the data structure that describes a single graph to run.
+type GraphConfig struct {
+	GraphConfigData
+	ResList []resources.Res
+}
+
+// UnmarshalYAML unmarshalls the complete graph.
+func (c *GraphConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	// Unmarshal the graph data, except the resources
+	if err := unmarshal(&c.GraphConfigData); err != nil {
+		return err
+	}
+
+	// Unmarshal resources
+	var list Resources
+	list.Resources = map[string][]Resource{}
+	if err := unmarshal(&list); err != nil {
+		return err
+	}
+
+	// Finish unmarshalling by giving to each resource its kind
+	// and store each resource in the graph
+	for kind, resList := range list.Resources {
+		for _, res := range resList {
+			err := res.Decode(kind)
+			if err != nil {
+				return err
+			}
+			c.ResList = append(c.ResList, res.resource)
+		}
+	}
+
+	return nil
+}
+
+// UnmarshalYAML is the first stage for unmarshaling of resources.
+func (r *Resource) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	r.unmarshal = unmarshal
+	return unmarshal(&r.ResourceData)
+}
+
+// Decode is the second stage for unmarshaling of resources (knowing their
+// kind).
+func (r *Resource) Decode(kind string) (err error) {
+	r.resource, err = resources.NewEmptyNamedResource(kind)
+	if err != nil {
+		return err
+	}
+
+	err = r.unmarshal(r.resource)
+	if err != nil {
+		return err
+	}
+
+	// Set resource name, meta and kind
+	r.resource.SetName(r.Name)
+	r.resource.SetKind(strings.ToLower(kind))
+	meta := r.resource.Meta()
+	*meta = r.Meta
+	return
+}
+
+// Parse parses a data stream into the graph structure.
+func (c *GraphConfig) Parse(data []byte) error {
+	if err := yaml.Unmarshal(data, c); err != nil {
+		return err
+	}
+	if c.Graph == "" {
+		return errors.New("graph config: invalid graph")
+	}
+	return nil
+}
+
+// NewGraphFromConfig transforms a GraphConfig struct into a new graph.
+// FIXME: remove any possibly left over, now obsolete graph diff code from here!
+func (c *GraphConfig) NewGraphFromConfig(hostname string, world resources.World, noop bool) (*pgraph.Graph, error) {
+	// hostname is the uuid for the host
+
+	var graph *pgraph.Graph          // new graph to return
+	graph = pgraph.NewGraph("Graph") // give graph a default name
+
+	var lookup = make(map[string]map[string]*pgraph.Vertex)
+
+	//log.Printf("%+v", config) // debug
+
+	// TODO: if defined (somehow)...
+	graph.SetName(c.Graph) // set graph name
+
+	var keep []*pgraph.Vertex        // list of vertex which are the same in new graph
+	var resourceList []resources.Res // list of resources to export
+
+	// Resources
+	for _, res := range c.ResList {
+		kind := res.Kind()
+		if _, exists := lookup[kind]; !exists {
+			lookup[kind] = make(map[string]*pgraph.Vertex)
+		}
+		// XXX: should we export based on a @@ prefix, or a metaparam
+		// like exported => true || exported => (host pattern)||(other pattern?)
+		if !strings.HasPrefix(res.GetName(), "@@") { // not exported resource
+			v := graph.CompareMatch(res)
+			if v == nil { // no match found
+				v = pgraph.NewVertex(res)
+				graph.AddVertex(v) // call standalone in case not part of an edge
+			}
+			lookup[kind][res.GetName()] = v // used for constructing edges
+			keep = append(keep, v)          // append
+
+		} else if !noop { // do not export any resources if noop
+			// store for addition to backend storage...
+			res.SetName(res.GetName()[2:]) // slice off @@
+			res.SetKind(kind)              // cheap init
+			resourceList = append(resourceList, res)
+		}
+	}
+
+	// store in backend (usually etcd)
+	if err := world.ResExport(resourceList); err != nil {
+		return nil, fmt.Errorf("Config: Could not export resources: %v", err)
+	}
+
+	// lookup from backend (usually etcd)
+	var hostnameFilter []string // empty to get from everyone
+	kindFilter := []string{}
+	for _, t := range c.Collector {
+		kind := strings.ToLower(t.Kind)
+		kindFilter = append(kindFilter, kind)
+	}
+	// do all the graph look ups in one single step, so that if the backend
+	// database changes, we don't have a partial state of affairs...
+	if len(kindFilter) > 0 { // if kindFilter is empty, don't need to do lookups!
+		var err error
+		resourceList, err = world.ResCollect(hostnameFilter, kindFilter)
+		if err != nil {
+			return nil, fmt.Errorf("Config: Could not collect resources: %v", err)
+		}
+	}
+	for _, res := range resourceList {
+		matched := false
+		// see if we find a collect pattern that matches
+		for _, t := range c.Collector {
+			kind := strings.ToLower(t.Kind)
+			// use t.Kind and optionally t.Pattern to collect from storage
+			log.Printf("Collect: %v; Pattern: %v", kind, t.Pattern)
+
+			// XXX: expand to more complex pattern matching here...
+			if res.Kind() != kind {
+				continue
+			}
+
+			if matched {
+				// we've already matched this resource, should we match again?
+				log.Printf("Config: Warning: Matching %s[%s] again!", kind, res.GetName())
+			}
+			matched = true
+
+			// collect resources but add the noop metaparam
+			//if noop { // now done in mgmtmain
+			//	res.Meta().Noop = noop
+			//}
+
+			if t.Pattern != "" { // XXX: simplistic for now
+				res.CollectPattern(t.Pattern) // res.Dirname = t.Pattern
+			}
+
+			log.Printf("Collect: %s[%s]: collected!", kind, res.GetName())
+
+			// XXX: similar to other resource add code:
+			if _, exists := lookup[kind]; !exists {
+				lookup[kind] = make(map[string]*pgraph.Vertex)
+			}
+			v := graph.CompareMatch(res)
+			if v == nil { // no match found
+				v = pgraph.NewVertex(res)
+				graph.AddVertex(v) // call standalone in case not part of an edge
+			}
+			lookup[kind][res.GetName()] = v // used for constructing edges
+			keep = append(keep, v)          // append
+
+			//break // let's see if another resource even matches
+		}
+	}
+
+	for _, e := range c.Edges {
+		if _, ok := lookup[strings.ToLower(e.From.Kind)]; !ok {
+			return nil, fmt.Errorf("can't find 'from' resource")
+		}
+		if _, ok := lookup[strings.ToLower(e.To.Kind)]; !ok {
+			return nil, fmt.Errorf("can't find 'to' resource")
+		}
+		if _, ok := lookup[strings.ToLower(e.From.Kind)][e.From.Name]; !ok {
+			return nil, fmt.Errorf("can't find 'from' name")
+		}
+		if _, ok := lookup[strings.ToLower(e.To.Kind)][e.To.Name]; !ok {
+			return nil, fmt.Errorf("can't find 'to' name")
+		}
+		from := lookup[strings.ToLower(e.From.Kind)][e.From.Name]
+		to := lookup[strings.ToLower(e.To.Kind)][e.To.Name]
+		edge := pgraph.NewEdge(e.Name)
+		edge.Notify = e.Notify
+		graph.AddEdge(from, to, edge)
+	}
+
+	return graph, nil
+}
+
+// ParseConfigFromFile takes a filename and returns the graph config structure.
+func ParseConfigFromFile(filename string) *GraphConfig {
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		log.Printf("Config: Error: ParseConfigFromFile: File: %v", err)
+		return nil
+	}
+
+	var config GraphConfig
+	if err := config.Parse(data); err != nil {
+		log.Printf("Config: Error: ParseConfigFromFile: Parse: %v", err)
+		return nil
+	}
+
+	return &config
+}