diff --git a/examples/graph0.hcl b/examples/graph0.hcl new file mode 100644 index 00000000..2e25f6e4 --- /dev/null +++ b/examples/graph0.hcl @@ -0,0 +1,20 @@ +resource "file" "file1" { + path = "/tmp/mgmt-hello-world" + content = "hello, world" + state = "exists" +} + +resource "noop" "noop1" { + test = "nil" +} + +edge "e1" { + from = { + kind = "noop" + name = "noop1" + } + to = { + kind = "file" + name = "file1" + } +} diff --git a/examples/graph1.hcl b/examples/graph1.hcl new file mode 100644 index 00000000..f4b5724f --- /dev/null +++ b/examples/graph1.hcl @@ -0,0 +1,4 @@ +resource "exec" "exec1" { + cmd = "cat /tmp/mgmt-hello-world" + state = "present" +} diff --git a/hcl/gapi.go b/hcl/gapi.go new file mode 100644 index 00000000..35b7dd31 --- /dev/null +++ b/hcl/gapi.go @@ -0,0 +1,155 @@ +// Mgmt +// Copyright (C) 2013-2017+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package hcl + +import ( + "fmt" + "log" + "sync" + + "github.com/purpleidea/mgmt/gapi" + "github.com/purpleidea/mgmt/pgraph" + "github.com/purpleidea/mgmt/recwatch" +) + +// GAPI ... +type GAPI struct { + File *string + + initialized bool + data gapi.Data + wg sync.WaitGroup + closeChan chan struct{} + configWatcher *recwatch.ConfigWatcher +} + +// NewGAPI ... +func NewGAPI(data gapi.Data, file *string) (*GAPI, error) { + if file == nil { + return nil, fmt.Errorf("empty file given") + } + + obj := &GAPI{ + File: file, + } + return obj, obj.Init(data) +} + +// Init ... +func (obj *GAPI) Init(d gapi.Data) error { + if obj.initialized { + return fmt.Errorf("already initialized") + } + + if obj.File == nil { + return fmt.Errorf("file cannot be nil") + } + + obj.data = d + obj.closeChan = make(chan struct{}) + obj.initialized = true + obj.configWatcher = recwatch.NewConfigWatcher() + + return nil +} + +// Graph ... +func (obj *GAPI) Graph() (*pgraph.Graph, error) { + config, err := loadHcl(obj.File) + if err != nil { + return nil, fmt.Errorf("unable to parse graph: %s", err) + } + + return graphFromConfig(config, obj.data) +} + +// Next ... +func (obj *GAPI) Next() chan gapi.Next { + ch := make(chan gapi.Next) + obj.wg.Add(1) + + go func() { + defer obj.wg.Done() + defer close(ch) + if !obj.initialized { + next := gapi.Next{ + Err: fmt.Errorf("hcl: GAPI is not initialized"), + Exit: true, + } + ch <- next + return + } + startChan := make(chan struct{}) // start signal + close(startChan) // kick it off! + + watchChan, configChan := make(chan error), make(chan error) + if obj.data.NoConfigWatch { + configChan = nil + } else { + configChan = obj.configWatcher.ConfigWatch(*obj.File) // simple + } + if obj.data.NoStreamWatch { + watchChan = nil + } else { + watchChan = obj.data.World.ResWatch() + } + + for { + var err error + var ok bool + + select { + case <-startChan: + startChan = nil + case err, ok = <-watchChan: + case err, ok = <-configChan: + if !ok { + return + } + case <-obj.closeChan: + return + } + + log.Printf("hcl: generating new graph") + next := gapi.Next{ + Err: err, + } + + select { + case ch <- next: + case <-obj.closeChan: + return + } + } + }() + + return ch +} + +// Close ... +func (obj *GAPI) Close() error { + if !obj.initialized { + return fmt.Errorf("hcl: GAPI is not initialized") + } + + obj.configWatcher.Close() + close(obj.closeChan) + obj.wg.Wait() + obj.initialized = false + return nil +} diff --git a/hcl/parse.go b/hcl/parse.go new file mode 100644 index 00000000..1d005150 --- /dev/null +++ b/hcl/parse.go @@ -0,0 +1,337 @@ +// Mgmt +// Copyright (C) 2013-2017+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package hcl + +import ( + "fmt" + "io/ioutil" + "log" + "strings" + + "github.com/hashicorp/hcl" + "github.com/hashicorp/hcl/hcl/ast" + "github.com/purpleidea/mgmt/gapi" + "github.com/purpleidea/mgmt/pgraph" + "github.com/purpleidea/mgmt/resources" +) + +type collectorResConfig struct { + Kind string + Pattern string +} + +// Config defines the structure of the hcl config. +type Config struct { + Resources []*Resource + Edges []*Edge + Collector []collectorResConfig +} + +// vertex is the data structure of a vertex. +type vertex struct { + Kind string `hcl:"kind"` + Name string `hcl:"name"` +} + +// Edge defines an edge in hcl. +type Edge struct { + Name string + From vertex + To vertex + Notify bool +} + +// Resources define the state for resources. +type Resources struct { + Resources []resources.Res +} + +// Resource ... +type Resource struct { + Name string + Kind string + resource resources.Res + Meta resources.MetaParams +} + +type key struct { + kind, name string +} + +func graphFromConfig(c *Config, data gapi.Data) (*pgraph.Graph, error) { + var graph *pgraph.Graph + var err error + + graph, err = pgraph.NewGraph("Graph") + if err != nil { + return nil, fmt.Errorf("unable to create graph from config: %s", err) + } + + lookup := make(map[key]pgraph.Vertex) + + var keep []pgraph.Vertex + var resourceList []resources.Res + + log.Printf("HCL: parsing %d resources", len(c.Resources)) + for _, r := range c.Resources { + res := r.resource + kind := r.resource.GetKind() + + log.Printf("HCL: resource \"%s\" \"%s\"", kind, r.Name) + if !strings.HasPrefix(res.GetName(), "@@") { + fn := func(v pgraph.Vertex) (bool, error) { + return resources.VtoR(v).Compare(res), nil + } + v, err := graph.VertexMatchFn(fn) + if err != nil { + return nil, fmt.Errorf("could not match vertex: %s", err) + } + if v == nil { + v = res + graph.AddVertex(v) + } + lookup[key{kind, res.GetName()}] = v + keep = append(keep, v) + } else if !data.Noop { + res.SetName(res.GetName()[2:]) + res.SetKind(kind) + resourceList = append(resourceList, res) + } + } + + // store in backend (usually etcd) + if err := data.World.ResExport(resourceList); err != nil { + return nil, fmt.Errorf("Config: Could not export resources: %v", err) + } + + // lookup from backend (usually etcd) + var hostnameFilter []string // empty to get from everyone + kindFilter := []string{} + for _, t := range c.Collector { + kind := strings.ToLower(t.Kind) + kindFilter = append(kindFilter, kind) + } + // do all the graph look ups in one single step, so that if the backend + // database changes, we don't have a partial state of affairs... + if len(kindFilter) > 0 { // if kindFilter is empty, don't need to do lookups! + var err error + resourceList, err = data.World.ResCollect(hostnameFilter, kindFilter) + if err != nil { + return nil, fmt.Errorf("Config: Could not collect resources: %v", err) + } + } + for _, res := range resourceList { + matched := false + // see if we find a collect pattern that matches + for _, t := range c.Collector { + kind := strings.ToLower(t.Kind) + // use t.Kind and optionally t.Pattern to collect from storage + log.Printf("Collect: %v; Pattern: %v", kind, t.Pattern) + + // XXX: expand to more complex pattern matching here... + if res.GetKind() != kind { + continue + } + + if matched { + // we've already matched this resource, should we match again? + log.Printf("Config: Warning: Matching %s again!", res) + } + matched = true + + // collect resources but add the noop metaparam + //if noop { // now done in mgmtmain + // res.Meta().Noop = noop + //} + + if t.Pattern != "" { // XXX: simplistic for now + res.CollectPattern(t.Pattern) // res.Dirname = t.Pattern + } + + log.Printf("Collect: %s: collected!", res) + + // XXX: similar to other resource add code: + // if _, exists := lookup[kind]; !exists { + // lookup[kind] = make(map[string]pgraph.Vertex) + // } + + fn := func(v pgraph.Vertex) (bool, error) { + return resources.VtoR(v).Compare(res), nil + } + v, err := graph.VertexMatchFn(fn) + if err != nil { + return nil, fmt.Errorf("could not VertexMatchFn() resource: %s", err) + } + if v == nil { // no match found + v = res // a standalone res can be a vertex + graph.AddVertex(v) // call standalone in case not part of an edge + } + lookup[key{kind, res.GetName()}] = v // used for constructing edges + keep = append(keep, v) // append + + //break // let's see if another resource even matches + } + } + + for _, e := range c.Edges { + if _, ok := lookup[key{strings.ToLower(e.From.Kind), e.From.Name}]; !ok { + return nil, fmt.Errorf("can't find 'from' name") + } + if _, ok := lookup[key{strings.ToLower(e.To.Kind), e.To.Name}]; !ok { + return nil, fmt.Errorf("can't find 'to' name") + } + from := lookup[key{strings.ToLower(e.From.Kind), e.From.Name}] + to := lookup[key{strings.ToLower(e.To.Kind), e.To.Name}] + edge := &resources.Edge{ + Name: e.Name, + Notify: e.Notify, + } + graph.AddEdge(from, to, edge) + } + + return graph, nil +} + +func loadHcl(f *string) (*Config, error) { + if f == nil { + return nil, fmt.Errorf("empty file given") + } + + log.Printf("loading file %s", *f) + data, err := ioutil.ReadFile(*f) + if err != nil { + return nil, fmt.Errorf("unable to read file: %v", err) + } + + log.Printf("parsing contents: %s", data) + file, err := hcl.ParseBytes(data) + if err != nil { + return nil, fmt.Errorf("unable to parse file: %s", err) + } + + config := new(Config) + + list, ok := file.Node.(*ast.ObjectList) + if !ok { + return nil, fmt.Errorf("unable to parse file: file does not contain root node object") + } + + if resources := list.Filter("resource"); len(resources.Items) > 0 { + var err error + config.Resources, err = loadResourcesHcl(resources) + if err != nil { + return nil, fmt.Errorf("unable to parse: %s", err) + } + } + + if edges := list.Filter("edge"); len(edges.Items) > 0 { + var err error + config.Edges, err = loadEdgesHcl(edges) + if err != nil { + return nil, fmt.Errorf("unable to parse: %s", err) + } + } + + return config, nil +} + +func loadEdgesHcl(list *ast.ObjectList) ([]*Edge, error) { + list = list.Children() + if len(list.Items) == 0 { + return nil, nil + } + + var result []*Edge + + for _, item := range list.Items { + name := item.Keys[0].Token.Value().(string) + + var config Edge + if err := hcl.DecodeObject(&config, item.Val); err != nil { + return nil, fmt.Errorf( + "Error reading config for %s: %s", + name, + err) + } + + config.Name = name + + result = append(result, &config) + } + return result, nil +} + +func loadResourcesHcl(list *ast.ObjectList) ([]*Resource, error) { + list = list.Children() + if len(list.Items) == 0 { + return nil, nil + } + + var result []*Resource + + log.Printf("HCLParse: parsing %d items", len(list.Items)) + for _, item := range list.Items { + kind := item.Keys[0].Token.Value().(string) + name := item.Keys[1].Token.Value().(string) + + var listVal *ast.ObjectList + if ot, ok := item.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return nil, fmt.Errorf("module '%s': should be an object", name) + } + + res, err := resources.NewResource(kind) + if err != nil { + log.Printf("HCLParse: unable to parse resource: %v", err) + return nil, err + } + + res.SetName(name) + + if err := hcl.DecodeObject(res, item.Val); err != nil { + log.Printf("HCLParse: unable to decode body: %v", err) + return nil, fmt.Errorf( + "Error reading config for %s: %s", + name, + err) + } + + var params = resources.DefaultMetaParams + if o := listVal.Filter("meta"); len(o.Items) > 0 { + err := hcl.DecodeObject(¶ms, o) + if err != nil { + return nil, fmt.Errorf( + "Error parsing meta for %s: %s", + name, + err) + } + } + + meta := res.Meta() + *meta = params + + result = append(result, &Resource{ + Name: name, + Kind: kind, + resource: res, + }) + } + + return result, nil +} diff --git a/lib/cli.go b/lib/cli.go index 559355b3..36964a6b 100644 --- a/lib/cli.go +++ b/lib/cli.go @@ -24,6 +24,7 @@ import ( "os/signal" "syscall" + "github.com/purpleidea/mgmt/hcl" "github.com/purpleidea/mgmt/puppet" "github.com/purpleidea/mgmt/yamlgraph" "github.com/purpleidea/mgmt/yamlgraph2" @@ -89,6 +90,14 @@ func run(c *cli.Context) error { PuppetConf: c.String("puppet-conf"), } } + if h := c.String("hcl"); c.IsSet("hcl") { + if obj.GAPI != nil { + return fmt.Errorf("can't combine hcl GAPI with existing GAPI") + } + obj.GAPI = &hcl.GAPI{ + File: &h, + } + } obj.Remotes = c.StringSlice("remote") // FIXME: GAPI-ify somehow? obj.NoWatch = c.Bool("no-watch") @@ -222,6 +231,11 @@ func CLI(program, version string, flags Flags) error { Value: "", Usage: "yaml graph definition to run (parser v2)", }, + cli.StringFlag{ + Name: "hcl", + Value: "", + Usage: "hcl graph definition to run", + }, cli.StringFlag{ Name: "puppet, p", Value: "",