diff --git a/hcl/gapi.go b/hcl/gapi.go deleted file mode 100644 index 50e4646b..00000000 --- a/hcl/gapi.go +++ /dev/null @@ -1,198 +0,0 @@ -// Mgmt -// Copyright (C) 2013-2018+ James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -package hcl - -import ( - "fmt" - "log" - "sync" - - "github.com/purpleidea/mgmt/gapi" - "github.com/purpleidea/mgmt/pgraph" - "github.com/purpleidea/mgmt/resources" - - errwrap "github.com/pkg/errors" - "github.com/urfave/cli" -) - -const ( - // Name is the name of this frontend. - Name = "hcl" - // Start is the entry point filename that we use. It is arbitrary. - Start = "/start.hcl" -) - -func init() { - gapi.Register(Name, func() gapi.GAPI { return &GAPI{} }) // register -} - -// GAPI ... -type GAPI struct { - InputURI string - - initialized bool - data gapi.Data - wg sync.WaitGroup - closeChan chan struct{} -} - -// Cli takes a cli.Context, and returns our GAPI if activated. All arguments -// should take the prefix of the registered name. On activation, if there are -// any validation problems, you should return an error. If this was not -// activated, then you should return a nil GAPI and a nil error. -func (obj *GAPI) Cli(c *cli.Context, fs resources.Fs) (*gapi.Deploy, error) { - if s := c.String(Name); c.IsSet(Name) { - if s == "" { - return nil, fmt.Errorf("%s input is empty", Name) - } - - // TODO: single file input for now - if err := gapi.CopyFileToFs(fs, s, Start); err != nil { - return nil, errwrap.Wrapf(err, "can't copy code from `%s` to `%s`", s, Start) - } - - return &gapi.Deploy{ - Name: Name, - Noop: c.GlobalBool("noop"), - Sema: c.GlobalInt("sema"), - GAPI: &GAPI{ - InputURI: fs.URI(), - // TODO: add properties here... - }, - }, nil - } - return nil, nil // we weren't activated! -} - -// CliFlags returns a list of flags used by this deploy subcommand. -func (obj *GAPI) CliFlags() []cli.Flag { - return []cli.Flag{ - cli.StringFlag{ - Name: fmt.Sprintf("%s", Name), - Value: "", - Usage: "hcl graph definition to run", - }, - } -} - -// Init ... -func (obj *GAPI) Init(d gapi.Data) error { - if obj.initialized { - return fmt.Errorf("already initialized") - } - if obj.InputURI == "" { - return fmt.Errorf("the InputURI param must be specified") - } - obj.data = d - obj.closeChan = make(chan struct{}) - obj.initialized = true - return nil -} - -// Graph ... -func (obj *GAPI) Graph() (*pgraph.Graph, error) { - if !obj.initialized { - return nil, fmt.Errorf("%s: GAPI is not initialized", Name) - } - - fs, err := obj.data.World.Fs(obj.InputURI) // open the remote file system - if err != nil { - return nil, errwrap.Wrapf(err, "can't load code from file system `%s`", obj.InputURI) - } - - b, err := fs.ReadFile(Start) // read the single file out of it - if err != nil { - return nil, errwrap.Wrapf(err, "can't read code from file `%s`", Start) - } - - config, err := loadHcl(b) - if err != nil { - return nil, fmt.Errorf("unable to parse graph: %s", err) - } - - return graphFromConfig(config, obj.data) -} - -// Next ... -func (obj *GAPI) Next() chan gapi.Next { - ch := make(chan gapi.Next) - obj.wg.Add(1) - - go func() { - defer obj.wg.Done() - defer close(ch) - if !obj.initialized { - next := gapi.Next{ - Err: fmt.Errorf("%s: GAPI is not initialized", Name), - Exit: true, - } - ch <- next - return - } - startChan := make(chan struct{}) // start signal - close(startChan) // kick it off! - - var watchChan chan error - if obj.data.NoStreamWatch { - watchChan = nil - } else { - watchChan = obj.data.World.ResWatch() - } - - for { - var err error - var ok bool - - select { - case <-startChan: - startChan = nil - case err, ok = <-watchChan: - if !ok { - return - } - case <-obj.closeChan: - return - } - - log.Printf("%s: generating new graph", Name) - next := gapi.Next{ - Err: err, - } - - select { - case ch <- next: - case <-obj.closeChan: - return - } - } - }() - - return ch -} - -// Close ... -func (obj *GAPI) Close() error { - if !obj.initialized { - return fmt.Errorf("%s: GAPI is not initialized", Name) - } - - close(obj.closeChan) - obj.wg.Wait() - obj.initialized = false - return nil -} diff --git a/hcl/hil/interpolate.go b/hcl/hil/interpolate.go deleted file mode 100644 index bb015ee3..00000000 --- a/hcl/hil/interpolate.go +++ /dev/null @@ -1,89 +0,0 @@ -// Mgmt -// Copyright (C) 2013-2018+ James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -package hil - -import ( - "fmt" - "strings" - - "github.com/hashicorp/hil/ast" -) - -// Variable defines an interpolated variable. -type Variable interface { - Key() string -} - -// ResourceVariable defines a variable type used to reference fields of a resource -// e.g. ${file.file1.Content} -type ResourceVariable struct { - Kind, Name, Field string -} - -// Key returns a string representation of the variable key. -func (r *ResourceVariable) Key() string { - return fmt.Sprintf("%s.%s.%s", r.Kind, r.Name, r.Field) -} - -// NewInterpolatedVariable takes a variable key and return the interpolated variable -// of the required type. -func NewInterpolatedVariable(k string) (Variable, error) { - // for now resource variables are the only thing. - parts := strings.SplitN(k, ".", 3) - - return &ResourceVariable{ - Kind: parts[0], - Name: parts[1], - Field: parts[2], - }, nil -} - -// ParseVariables will traverse a HIL tree looking for variables and returns a -// list of them. -func ParseVariables(tree ast.Node) ([]Variable, error) { - var result []Variable - var finalErr error - - visitor := func(n ast.Node) ast.Node { - if finalErr != nil { - return n - } - - switch nt := n.(type) { - case *ast.VariableAccess: - v, err := NewInterpolatedVariable(nt.Name) - if err != nil { - finalErr = err - return n - } - result = append(result, v) - default: - return n - } - - return n - } - - tree.Accept(visitor) - - if finalErr != nil { - return nil, finalErr - } - - return result, nil -} diff --git a/hcl/parse.go b/hcl/parse.go deleted file mode 100644 index 832d89cc..00000000 --- a/hcl/parse.go +++ /dev/null @@ -1,381 +0,0 @@ -// Mgmt -// Copyright (C) 2013-2018+ James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -package hcl - -import ( - "fmt" - "log" - "strings" - - "github.com/hashicorp/hcl" - "github.com/hashicorp/hcl/hcl/ast" - "github.com/hashicorp/hil" - "github.com/purpleidea/mgmt/gapi" - hv "github.com/purpleidea/mgmt/hcl/hil" - "github.com/purpleidea/mgmt/pgraph" - "github.com/purpleidea/mgmt/resources" -) - -type collectorResConfig struct { - Kind string - Pattern string -} - -// Config defines the structure of the hcl config. -type Config struct { - Resources []*Resource - Edges []*Edge - Collector []collectorResConfig -} - -// vertex is the data structure of a vertex. -type vertex struct { - Kind string `hcl:"kind"` - Name string `hcl:"name"` -} - -// Edge defines an edge in hcl. -type Edge struct { - Name string - From vertex - To vertex - Notify bool -} - -// Resources define the state for resources. -type Resources struct { - Resources []resources.Res -} - -// Resource ... -type Resource struct { - Name string - Kind string - resource resources.Res - Meta resources.MetaParams - deps []*Edge - rcv map[string]*hv.ResourceVariable -} - -type key struct { - kind, name string -} - -func graphFromConfig(c *Config, data gapi.Data) (*pgraph.Graph, error) { - var graph *pgraph.Graph - var err error - - graph, err = pgraph.NewGraph("Graph") - if err != nil { - return nil, fmt.Errorf("unable to create graph from config: %s", err) - } - - lookup := make(map[key]pgraph.Vertex) - - var keep []pgraph.Vertex - var resourceList []resources.Res - - log.Printf("hcl: parsing %d resources", len(c.Resources)) - for _, r := range c.Resources { - res := r.resource - kind := r.resource.GetKind() - - log.Printf("hcl: resource \"%s\" \"%s\"", kind, r.Name) - if !strings.HasPrefix(res.GetName(), "@@") { - fn := func(v pgraph.Vertex) (bool, error) { - return resources.VtoR(v).Compare(res), nil - } - v, err := graph.VertexMatchFn(fn) - if err != nil { - return nil, fmt.Errorf("could not match vertex: %s", err) - } - if v == nil { - v = res - graph.AddVertex(v) - } - lookup[key{kind, res.GetName()}] = v - keep = append(keep, v) - } else if !data.Noop { - res.SetName(res.GetName()[2:]) - res.SetKind(kind) - resourceList = append(resourceList, res) - } - } - - // store in backend (usually etcd) - if err := data.World.ResExport(resourceList); err != nil { - return nil, fmt.Errorf("Config: Could not export resources: %v", err) - } - - // lookup from backend (usually etcd) - var hostnameFilter []string // empty to get from everyone - kindFilter := []string{} - for _, t := range c.Collector { - kind := strings.ToLower(t.Kind) - kindFilter = append(kindFilter, kind) - } - // do all the graph look ups in one single step, so that if the backend - // database changes, we don't have a partial state of affairs... - if len(kindFilter) > 0 { // if kindFilter is empty, don't need to do lookups! - var err error - resourceList, err = data.World.ResCollect(hostnameFilter, kindFilter) - if err != nil { - return nil, fmt.Errorf("Config: Could not collect resources: %v", err) - } - } - for _, res := range resourceList { - matched := false - // see if we find a collect pattern that matches - for _, t := range c.Collector { - kind := strings.ToLower(t.Kind) - // use t.Kind and optionally t.Pattern to collect from storage - log.Printf("Collect: %v; Pattern: %v", kind, t.Pattern) - - // XXX: expand to more complex pattern matching here... - if res.GetKind() != kind { - continue - } - - if matched { - // we've already matched this resource, should we match again? - log.Printf("Config: Warning: Matching %s again!", res) - } - matched = true - - // collect resources but add the noop metaparam - //if noop { // now done in mgmtmain - // res.Meta().Noop = noop - //} - - if t.Pattern != "" { // XXX: simplistic for now - res.CollectPattern(t.Pattern) // res.Dirname = t.Pattern - } - - log.Printf("Collect: %s: collected!", res) - - // XXX: similar to other resource add code: - // if _, exists := lookup[kind]; !exists { - // lookup[kind] = make(map[string]pgraph.Vertex) - // } - - fn := func(v pgraph.Vertex) (bool, error) { - return resources.VtoR(v).Compare(res), nil - } - v, err := graph.VertexMatchFn(fn) - if err != nil { - return nil, fmt.Errorf("could not VertexMatchFn() resource: %s", err) - } - if v == nil { // no match found - v = res // a standalone res can be a vertex - graph.AddVertex(v) // call standalone in case not part of an edge - } - lookup[key{kind, res.GetName()}] = v // used for constructing edges - keep = append(keep, v) // append - - //break // let's see if another resource even matches - } - } - - for _, r := range c.Resources { - for _, e := range r.deps { - if _, ok := lookup[key{strings.ToLower(e.From.Kind), e.From.Name}]; !ok { - return nil, fmt.Errorf("can't find 'from' name") - } - if _, ok := lookup[key{strings.ToLower(e.To.Kind), e.To.Name}]; !ok { - return nil, fmt.Errorf("can't find 'to' name") - } - from := lookup[key{strings.ToLower(e.From.Kind), e.From.Name}] - to := lookup[key{strings.ToLower(e.To.Kind), e.To.Name}] - edge := &resources.Edge{ - Name: e.Name, - Notify: e.Notify, - } - graph.AddEdge(from, to, edge) - } - - recv := make(map[string]*resources.Send) - // build Rcv's from resource variables - for k, v := range r.rcv { - send, ok := lookup[key{strings.ToLower(v.Kind), v.Name}] - if !ok { - return nil, fmt.Errorf("resource not found") - } - - recv[strings.ToUpper(string(k[0]))+k[1:]] = &resources.Send{ - Res: resources.VtoR(send), - Key: v.Field, - } - - to := lookup[key{strings.ToLower(r.Kind), r.Name}] - edge := &resources.Edge{ - Name: v.Name, - Notify: true, - } - graph.AddEdge(send, to, edge) - } - - r.resource.SetRecv(recv) - } - - return graph, nil -} - -func loadHcl(data []byte) (*Config, error) { - if len(data) == 0 { - return nil, fmt.Errorf("empty data given") - } - - file, err := hcl.ParseBytes(data) - if err != nil { - return nil, fmt.Errorf("unable to parse file: %s", err) - } - - config := new(Config) - - list, ok := file.Node.(*ast.ObjectList) - if !ok { - return nil, fmt.Errorf("unable to parse file: file does not contain root node object") - } - - if resources := list.Filter("resource"); len(resources.Items) > 0 { - var err error - config.Resources, err = loadResourcesHcl(resources) - if err != nil { - return nil, fmt.Errorf("unable to parse: %s", err) - } - } - - return config, nil -} - -func loadResourcesHcl(list *ast.ObjectList) ([]*Resource, error) { - list = list.Children() - if len(list.Items) == 0 { - return nil, nil - } - - var result []*Resource - - for _, item := range list.Items { - kind := item.Keys[0].Token.Value().(string) - name := item.Keys[1].Token.Value().(string) - - var listVal *ast.ObjectList - if ot, ok := item.Val.(*ast.ObjectType); ok { - listVal = ot.List - } else { - return nil, fmt.Errorf("module '%s': should be an object", name) - } - - var params = resources.DefaultMetaParams - if o := listVal.Filter("meta"); len(o.Items) > 0 { - err := hcl.DecodeObject(¶ms, o) - if err != nil { - return nil, fmt.Errorf( - "Error parsing meta for %s: %s", - name, - err) - } - } - - var deps []string - if edges := listVal.Filter("depends_on"); len(edges.Items) > 0 { - err := hcl.DecodeObject(&deps, edges.Items[0].Val) - if err != nil { - return nil, fmt.Errorf("unable to parse: %s", err) - } - } - - var edges []*Edge - for _, dep := range deps { - vertices := strings.Split(dep, ".") - edges = append(edges, &Edge{ - To: vertex{ - Kind: kind, - Name: name, - }, - From: vertex{ - Kind: vertices[0], - Name: vertices[1], - }, - }) - } - - var config map[string]interface{} - if err := hcl.DecodeObject(&config, item.Val); err != nil { - log.Printf("hcl: unable to decode body: %v", err) - return nil, fmt.Errorf( - "Error reading config for %s: %s", - name, - err) - } - - delete(config, "meta") - delete(config, "depends_on") - - rcv := make(map[string]*hv.ResourceVariable) - // parse strings for hil - for k, v := range config { - n, err := hil.Parse(v.(string)) - if err != nil { - return nil, fmt.Errorf("unable to parse fields: %v", err) - } - - variables, err := hv.ParseVariables(n) - if err != nil { - return nil, fmt.Errorf("unable to parse variables: %v", err) - } - - for _, v := range variables { - val, ok := v.(*hv.ResourceVariable) - if !ok { - continue - } - - rcv[k] = val - } - } - - res, err := resources.NewNamedResource(kind, name) - if err != nil { - log.Printf("hcl: unable to parse resource: %v", err) - return nil, err - } - - if err := hcl.DecodeObject(res, item.Val); err != nil { - log.Printf("hcl: unable to decode body: %v", err) - return nil, fmt.Errorf( - "Error reading config for %s: %s", - name, - err) - } - - meta := res.Meta() - *meta = params - - result = append(result, &Resource{ - Name: name, - Kind: kind, - resource: res, - deps: edges, - rcv: rcv, - }) - } - - return result, nil -} diff --git a/lib/deploy.go b/lib/deploy.go index 942ba395..c1c11840 100644 --- a/lib/deploy.go +++ b/lib/deploy.go @@ -27,7 +27,6 @@ import ( "github.com/purpleidea/mgmt/gapi" // these imports are so that GAPIs register themselves in init() - _ "github.com/purpleidea/mgmt/hcl" _ "github.com/purpleidea/mgmt/lang" _ "github.com/purpleidea/mgmt/puppet" _ "github.com/purpleidea/mgmt/yamlgraph"