lourenco/mgmt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lib: Remove hcl GAPI frontend

Browse Source 
This is currently unmaintained and the normal mcl language exists which
is preferable to this. As a result, I'm removing this for now to make an
upcoming refactor easier. We can add it back easily if someone has
interest.
This commit is contained in:
James Shubin
2018-03-13 12:57:07 -04:00
parent 754480a9b6
commit 6e0dfdb16f
4 changed files with 0 additions and 669 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
hcl/gapi.go
View File

@@ -1,198 +0,0 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package hcl
import (
"fmt"
"log"
"sync"
"github.com/purpleidea/mgmt/gapi"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
errwrap "github.com/pkg/errors"
"github.com/urfave/cli"
)
const (
// Name is the name of this frontend.
Name = "hcl"
// Start is the entry point filename that we use. It is arbitrary.
Start = "/start.hcl"
)
func init() {
gapi.Register(Name, func() gapi.GAPI { return &GAPI{} }) // register
}
// GAPI ...
type GAPI struct {
InputURI string
initialized bool
data gapi.Data
wg sync.WaitGroup
closeChan chan struct{}
}
// Cli takes a cli.Context, and returns our GAPI if activated. All arguments
// should take the prefix of the registered name. On activation, if there are
// any validation problems, you should return an error. If this was not
// activated, then you should return a nil GAPI and a nil error.
func (obj *GAPI) Cli(c *cli.Context, fs resources.Fs) (*gapi.Deploy, error) {
if s := c.String(Name); c.IsSet(Name) {
if s == "" {
return nil, fmt.Errorf("%s input is empty", Name)
}
// TODO: single file input for now
if err := gapi.CopyFileToFs(fs, s, Start); err != nil {
return nil, errwrap.Wrapf(err, "can't copy code from `%s` to `%s`", s, Start)
}
return &gapi.Deploy{
Name: Name,
Noop: c.GlobalBool("noop"),
Sema: c.GlobalInt("sema"),
GAPI: &GAPI{
InputURI: fs.URI(),
// TODO: add properties here...
},
}, nil
}
return nil, nil // we weren't activated!
}
// CliFlags returns a list of flags used by this deploy subcommand.
func (obj *GAPI) CliFlags() []cli.Flag {
return []cli.Flag{
cli.StringFlag{
Name: fmt.Sprintf("%s", Name),
Value: "",
Usage: "hcl graph definition to run",
},
}
}
// Init ...
func (obj *GAPI) Init(d gapi.Data) error {
if obj.initialized {
return fmt.Errorf("already initialized")
}
if obj.InputURI == "" {
return fmt.Errorf("the InputURI param must be specified")
}
obj.data = d
obj.closeChan = make(chan struct{})
obj.initialized = true
return nil
}
// Graph ...
func (obj *GAPI) Graph() (*pgraph.Graph, error) {
if !obj.initialized {
return nil, fmt.Errorf("%s: GAPI is not initialized", Name)
}
fs, err := obj.data.World.Fs(obj.InputURI) // open the remote file system
if err != nil {
return nil, errwrap.Wrapf(err, "can't load code from file system `%s`", obj.InputURI)
}
b, err := fs.ReadFile(Start) // read the single file out of it
if err != nil {
return nil, errwrap.Wrapf(err, "can't read code from file `%s`", Start)
}
config, err := loadHcl(b)
if err != nil {
return nil, fmt.Errorf("unable to parse graph: %s", err)
}
return graphFromConfig(config, obj.data)
}
// Next ...
func (obj *GAPI) Next() chan gapi.Next {
ch := make(chan gapi.Next)
obj.wg.Add(1)
go func() {
defer obj.wg.Done()
defer close(ch)
if !obj.initialized {
next := gapi.Next{
Err: fmt.Errorf("%s: GAPI is not initialized", Name),
Exit: true,
}
ch <- next
return
}
startChan := make(chan struct{}) // start signal
close(startChan) // kick it off!
var watchChan chan error
if obj.data.NoStreamWatch {
watchChan = nil
} else {
watchChan = obj.data.World.ResWatch()
}
for {
var err error
var ok bool
select {
case <-startChan:
startChan = nil
case err, ok = <-watchChan:
if !ok {
return
}
case <-obj.closeChan:
return
}
log.Printf("%s: generating new graph", Name)
next := gapi.Next{
Err: err,
}
select {
case ch <- next:
case <-obj.closeChan:
return
}
}
}()
return ch
}
// Close ...
func (obj *GAPI) Close() error {
if !obj.initialized {
return fmt.Errorf("%s: GAPI is not initialized", Name)
}
close(obj.closeChan)
obj.wg.Wait()
obj.initialized = false
return nil
}

89
hcl/hil/interpolate.go
View File

@@ -1,89 +0,0 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package hil
import (
"fmt"
"strings"
"github.com/hashicorp/hil/ast"
)
// Variable defines an interpolated variable.
type Variable interface {
Key() string
}
// ResourceVariable defines a variable type used to reference fields of a resource
// e.g. ${file.file1.Content}
type ResourceVariable struct {
Kind, Name, Field string
}
// Key returns a string representation of the variable key.
func (r *ResourceVariable) Key() string {
return fmt.Sprintf("%s.%s.%s", r.Kind, r.Name, r.Field)
}
// NewInterpolatedVariable takes a variable key and return the interpolated variable
// of the required type.
func NewInterpolatedVariable(k string) (Variable, error) {
// for now resource variables are the only thing.
parts := strings.SplitN(k, ".", 3)
return &ResourceVariable{
Kind: parts[0],
Name: parts[1],
Field: parts[2],
}, nil
}
// ParseVariables will traverse a HIL tree looking for variables and returns a
// list of them.
func ParseVariables(tree ast.Node) ([]Variable, error) {
var result []Variable
var finalErr error
visitor := func(n ast.Node) ast.Node {
if finalErr != nil {
return n
}
switch nt := n.(type) {
case *ast.VariableAccess:
v, err := NewInterpolatedVariable(nt.Name)
if err != nil {
finalErr = err
return n
}
result = append(result, v)
default:
return n
}
return n
}
tree.Accept(visitor)
if finalErr != nil {
return nil, finalErr
}
return result, nil
}

381
hcl/parse.go
View File

@@ -1,381 +0,0 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package hcl
import (
"fmt"
"log"
"strings"
"github.com/hashicorp/hcl"
"github.com/hashicorp/hcl/hcl/ast"
"github.com/hashicorp/hil"
"github.com/purpleidea/mgmt/gapi"
hv "github.com/purpleidea/mgmt/hcl/hil"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
)
type collectorResConfig struct {
Kind string
Pattern string
}
// Config defines the structure of the hcl config.
type Config struct {
Resources []*Resource
Edges []*Edge
Collector []collectorResConfig
}
// vertex is the data structure of a vertex.
type vertex struct {
Kind string `hcl:"kind"`
Name string `hcl:"name"`
}
// Edge defines an edge in hcl.
type Edge struct {
Name string
From vertex
To vertex
Notify bool
}
// Resources define the state for resources.
type Resources struct {
Resources []resources.Res
}
// Resource ...
type Resource struct {
Name string
Kind string
resource resources.Res
Meta resources.MetaParams
deps []*Edge
rcv map[string]*hv.ResourceVariable
}
type key struct {
kind, name string
}
func graphFromConfig(c *Config, data gapi.Data) (*pgraph.Graph, error) {
var graph *pgraph.Graph
var err error
graph, err = pgraph.NewGraph("Graph")
if err != nil {
return nil, fmt.Errorf("unable to create graph from config: %s", err)
}
lookup := make(map[key]pgraph.Vertex)
var keep []pgraph.Vertex
var resourceList []resources.Res
log.Printf("hcl: parsing %d resources", len(c.Resources))
for _, r := range c.Resources {
res := r.resource
kind := r.resource.GetKind()
log.Printf("hcl: resource \"%s\" \"%s\"", kind, r.Name)
if !strings.HasPrefix(res.GetName(), "@@") {
fn := func(v pgraph.Vertex) (bool, error) {
return resources.VtoR(v).Compare(res), nil
}
v, err := graph.VertexMatchFn(fn)
if err != nil {
return nil, fmt.Errorf("could not match vertex: %s", err)
}
if v == nil {
v = res
graph.AddVertex(v)
}
lookup[key{kind, res.GetName()}] = v
keep = append(keep, v)
} else if !data.Noop {
res.SetName(res.GetName()[2:])
res.SetKind(kind)
resourceList = append(resourceList, res)
}
}
// store in backend (usually etcd)
if err := data.World.ResExport(resourceList); err != nil {
return nil, fmt.Errorf("Config: Could not export resources: %v", err)
}
// lookup from backend (usually etcd)
var hostnameFilter []string // empty to get from everyone
kindFilter := []string{}
for _, t := range c.Collector {
kind := strings.ToLower(t.Kind)
kindFilter = append(kindFilter, kind)
}
// do all the graph look ups in one single step, so that if the backend
// database changes, we don't have a partial state of affairs...
if len(kindFilter) > 0 { // if kindFilter is empty, don't need to do lookups!
var err error
resourceList, err = data.World.ResCollect(hostnameFilter, kindFilter)
if err != nil {
return nil, fmt.Errorf("Config: Could not collect resources: %v", err)
}
}
for _, res := range resourceList {
matched := false
// see if we find a collect pattern that matches
for _, t := range c.Collector {
kind := strings.ToLower(t.Kind)
// use t.Kind and optionally t.Pattern to collect from storage
log.Printf("Collect: %v; Pattern: %v", kind, t.Pattern)
// XXX: expand to more complex pattern matching here...
if res.GetKind() != kind {
continue
}
if matched {
// we've already matched this resource, should we match again?
log.Printf("Config: Warning: Matching %s again!", res)
}
matched = true
// collect resources but add the noop metaparam
//if noop { // now done in mgmtmain
// res.Meta().Noop = noop
//}
if t.Pattern != "" { // XXX: simplistic for now
res.CollectPattern(t.Pattern) // res.Dirname = t.Pattern
}
log.Printf("Collect: %s: collected!", res)
// XXX: similar to other resource add code:
// if _, exists := lookup[kind]; !exists {
// lookup[kind] = make(map[string]pgraph.Vertex)
// }
fn := func(v pgraph.Vertex) (bool, error) {
return resources.VtoR(v).Compare(res), nil
}
v, err := graph.VertexMatchFn(fn)
if err != nil {
return nil, fmt.Errorf("could not VertexMatchFn() resource: %s", err)
}
if v == nil { // no match found
v = res // a standalone res can be a vertex
graph.AddVertex(v) // call standalone in case not part of an edge
}
lookup[key{kind, res.GetName()}] = v // used for constructing edges
keep = append(keep, v) // append
//break // let's see if another resource even matches
}
}
for _, r := range c.Resources {
for _, e := range r.deps {
if _, ok := lookup[key{strings.ToLower(e.From.Kind), e.From.Name}]; !ok {
return nil, fmt.Errorf("can't find 'from' name")
}
if _, ok := lookup[key{strings.ToLower(e.To.Kind), e.To.Name}]; !ok {
return nil, fmt.Errorf("can't find 'to' name")
}
from := lookup[key{strings.ToLower(e.From.Kind), e.From.Name}]
to := lookup[key{strings.ToLower(e.To.Kind), e.To.Name}]
edge := &resources.Edge{
Name: e.Name,
Notify: e.Notify,
}
graph.AddEdge(from, to, edge)
}
recv := make(map[string]*resources.Send)
// build Rcv's from resource variables
for k, v := range r.rcv {
send, ok := lookup[key{strings.ToLower(v.Kind), v.Name}]
if !ok {
return nil, fmt.Errorf("resource not found")
}
recv[strings.ToUpper(string(k[0]))+k[1:]] = &resources.Send{
Res: resources.VtoR(send),
Key: v.Field,
}
to := lookup[key{strings.ToLower(r.Kind), r.Name}]
edge := &resources.Edge{
Name: v.Name,
Notify: true,
}
graph.AddEdge(send, to, edge)
}
r.resource.SetRecv(recv)
}
return graph, nil
}
func loadHcl(data []byte) (*Config, error) {
if len(data) == 0 {
return nil, fmt.Errorf("empty data given")
}
file, err := hcl.ParseBytes(data)
if err != nil {
return nil, fmt.Errorf("unable to parse file: %s", err)
}
config := new(Config)
list, ok := file.Node.(*ast.ObjectList)
if !ok {
return nil, fmt.Errorf("unable to parse file: file does not contain root node object")
}
if resources := list.Filter("resource"); len(resources.Items) > 0 {
var err error
config.Resources, err = loadResourcesHcl(resources)
if err != nil {
return nil, fmt.Errorf("unable to parse: %s", err)
}
}
return config, nil
}
func loadResourcesHcl(list *ast.ObjectList) ([]*Resource, error) {
list = list.Children()
if len(list.Items) == 0 {
return nil, nil
}
var result []*Resource
for _, item := range list.Items {
kind := item.Keys[0].Token.Value().(string)
name := item.Keys[1].Token.Value().(string)
var listVal *ast.ObjectList
if ot, ok := item.Val.(*ast.ObjectType); ok {
listVal = ot.List
} else {
return nil, fmt.Errorf("module '%s': should be an object", name)
}
var params = resources.DefaultMetaParams
if o := listVal.Filter("meta"); len(o.Items) > 0 {
err := hcl.DecodeObject(&params, o)
if err != nil {
return nil, fmt.Errorf(
"Error parsing meta for %s: %s",
name,
err)
}
}
var deps []string
if edges := listVal.Filter("depends_on"); len(edges.Items) > 0 {
err := hcl.DecodeObject(&deps, edges.Items[0].Val)
if err != nil {
return nil, fmt.Errorf("unable to parse: %s", err)
}
}
var edges []*Edge
for _, dep := range deps {
vertices := strings.Split(dep, ".")
edges = append(edges, &Edge{
To: vertex{
Kind: kind,
Name: name,
},
From: vertex{
Kind: vertices[0],
Name: vertices[1],
},
})
}
var config map[string]interface{}
if err := hcl.DecodeObject(&config, item.Val); err != nil {
log.Printf("hcl: unable to decode body: %v", err)
return nil, fmt.Errorf(
"Error reading config for %s: %s",
name,
err)
}
delete(config, "meta")
delete(config, "depends_on")
rcv := make(map[string]*hv.ResourceVariable)
// parse strings for hil
for k, v := range config {
n, err := hil.Parse(v.(string))
if err != nil {
return nil, fmt.Errorf("unable to parse fields: %v", err)
}
variables, err := hv.ParseVariables(n)
if err != nil {
return nil, fmt.Errorf("unable to parse variables: %v", err)
}
for _, v := range variables {
val, ok := v.(*hv.ResourceVariable)
if !ok {
continue
}
rcv[k] = val
}
}
res, err := resources.NewNamedResource(kind, name)
if err != nil {
log.Printf("hcl: unable to parse resource: %v", err)
return nil, err
}
if err := hcl.DecodeObject(res, item.Val); err != nil {
log.Printf("hcl: unable to decode body: %v", err)
return nil, fmt.Errorf(
"Error reading config for %s: %s",
name,
err)
}
meta := res.Meta()
*meta = params
result = append(result, &Resource{
Name: name,
Kind: kind,
resource: res,
deps: edges,
rcv: rcv,
})
}
return result, nil
}

1
lib/deploy.go
View File

@@ -27,7 +27,6 @@ import (
"github.com/purpleidea/mgmt/gapi"
// these imports are so that GAPIs register themselves in init()
_ "github.com/purpleidea/mgmt/hcl"
_ "github.com/purpleidea/mgmt/lang"
_ "github.com/purpleidea/mgmt/puppet"
_ "github.com/purpleidea/mgmt/yamlgraph"