lourenco/mgmt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lang: Initial implementation of the mgmt language

Browse Source 
This is an initial implementation of the mgmt language. It is a
declarative (immutable) functional, reactive, domain specific
programming language. It is intended to be a language that is:

* safe
* powerful
* easy to reason about

With these properties, we hope this language, and the mgmt engine will
allow you to model the real-time systems that you'd like to automate.

This also includes a number of other associated changes. Sorry for the
large size of this patch.
This commit is contained in:
James Shubin
2018-01-20 08:09:29 -05:00
parent 1c8c0b2915
commit b19583e7d3
237 changed files with 25256 additions and 743 deletions
Download Patch File Download Diff File Expand all files Collapse all files

233
puppet/gapi.go
View File

@@ -19,32 +19,139 @@ package puppet
import (
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"sync"
"time"
"github.com/purpleidea/mgmt/gapi"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
"github.com/purpleidea/mgmt/util"
errwrap "github.com/pkg/errors"
"github.com/urfave/cli"
)
const (
// Name is the name of this frontend.
Name = "puppet"
// PuppetFile is the entry point filename that we use. It is arbitrary.
PuppetFile = "/file.pp"
// PuppetConf is the entry point config filename that we use.
PuppetConf = "/puppet.conf"
// PuppetSite is the entry point folder that we use.
PuppetSite = "/puppet/"
)
func init() {
gapi.Register(Name, func() gapi.GAPI { return &GAPI{} }) // register
}
// GAPI implements the main puppet GAPI interface.
type GAPI struct {
PuppetParam *string // puppet mode to run; nil if undefined
PuppetConf string // the path to an alternate puppet.conf file
InputURI string
Mode string // agent, file, string, dir
data gapi.Data
initialized bool
closeChan chan struct{}
wg sync.WaitGroup // sync group for tunnel go routines
puppetFile string
puppetString string
puppetDir string
puppetConf string // the path to an alternate puppet.conf file
data gapi.Data
initialized bool
closeChan chan struct{}
wg sync.WaitGroup // sync group for tunnel go routines
}
// NewGAPI creates a new puppet GAPI struct and calls Init().
func NewGAPI(data gapi.Data, puppetParam *string, puppetConf string) (*GAPI, error) {
obj := &GAPI{
PuppetParam: puppetParam,
PuppetConf: puppetConf,
// Cli takes a cli.Context, and returns our GAPI if activated. All arguments
// should take the prefix of the registered name. On activation, if there are
// any validation problems, you should return an error. If this was not
// activated, then you should return a nil GAPI and a nil error.
func (obj *GAPI) Cli(c *cli.Context, fs resources.Fs) (*gapi.Deploy, error) {
if s := c.String(Name); c.IsSet(Name) {
if s == "" {
return nil, fmt.Errorf("%s input is empty", Name)
}
isDir := func(p string) (bool, error) {
if !strings.HasPrefix(p, "/") {
return false, nil
}
if !strings.HasSuffix(s, "/") {
return false, nil
}
fi, err := os.Stat(p)
if err != nil {
return false, err
}
return fi.IsDir(), nil
}
var mode string
if s == "agent" {
mode = "agent"
} else if strings.HasSuffix(s, ".pp") {
mode = "file"
if err := gapi.CopyFileToFs(fs, s, PuppetFile); err != nil {
return nil, errwrap.Wrapf(err, "can't copy code from `%s` to `%s`", s, PuppetFile)
}
} else if exists, err := isDir(s); err != nil {
return nil, errwrap.Wrapf(err, "can't read dir `%s`", s)
} else if err == nil && exists { // from the isDir result...
// we have a whole directory of files to run
mode = "dir"
// TODO: this code path is untested! test and then rm this notice
if err := gapi.CopyDirToFs(fs, s, PuppetSite); err != nil {
return nil, errwrap.Wrapf(err, "can't copy code to `%s`", PuppetSite)
}
} else {
mode = "string"
if err := gapi.CopyStringToFs(fs, s, PuppetFile); err != nil {
return nil, errwrap.Wrapf(err, "can't copy code to `%s`", PuppetFile)
}
}
// TODO: do we want to include this if we have mode == "dir" ?
if pc := c.String("puppet-conf"); c.IsSet("puppet-conf") {
if err := gapi.CopyFileToFs(fs, pc, PuppetConf); err != nil {
return nil, errwrap.Wrapf(err, "can't copy puppet conf from `%s`")
}
}
return &gapi.Deploy{
Name: Name,
Noop: c.GlobalBool("noop"),
Sema: c.GlobalInt("sema"),
GAPI: &GAPI{
InputURI: fs.URI(),
Mode: mode,
// TODO: add properties here...
},
}, nil
}
return nil, nil // we weren't activated!
}
// CliFlags returns a list of flags used by this deploy subcommand.
func (obj *GAPI) CliFlags() []cli.Flag {
return []cli.Flag{
cli.StringFlag{
Name: "puppet, p",
Value: "",
Usage: "load graph from puppet, optionally takes a manifest or path to manifest file",
},
cli.StringFlag{
Name: "puppet-conf",
Value: "",
Usage: "the path to an alternate puppet.conf file",
},
}
return obj, obj.Init(data)
}
// Init initializes the puppet GAPI struct.
@@ -52,10 +159,83 @@ func (obj *GAPI) Init(data gapi.Data) error {
if obj.initialized {
return fmt.Errorf("already initialized")
}
if obj.PuppetParam == nil {
return fmt.Errorf("the PuppetParam param must be specified")
if obj.InputURI == "" {
return fmt.Errorf("the InputURI param must be specified")
}
switch obj.Mode {
case "agent", "file", "string", "dir":
// pass
default:
return fmt.Errorf("the Mode param is invalid")
}
obj.data = data // store for later
fs, err := obj.data.World.Fs(obj.InputURI) // open the remote file system
if err != nil {
return errwrap.Wrapf(err, "can't load data from file system `%s`", obj.InputURI)
}
if obj.Mode == "file" {
b, err := fs.ReadFile(PuppetFile) // read the single file out of it
if err != nil {
return errwrap.Wrapf(err, "can't read code from file `%s`", PuppetFile)
}
// store the puppet file on disk for other binaries to see and use
prefix := fmt.Sprintf("%s-%s-%s", data.Program, data.Hostname, strings.Replace(PuppetFile, "/", "", -1))
tmpfile, err := ioutil.TempFile("", prefix)
if err != nil {
return errwrap.Wrapf(err, "can't create temp file")
}
obj.puppetFile = tmpfile.Name() // path to temp file
defer tmpfile.Close()
if _, err := tmpfile.Write(b); err != nil {
return errwrap.Wrapf(err, "can't write file")
}
} else if obj.Mode == "string" {
b, err := fs.ReadFile(PuppetFile) // read the single code string out of it
if err != nil {
return errwrap.Wrapf(err, "can't read code from file `%s`", PuppetFile)
}
obj.puppetString = string(b)
} else if obj.Mode == "dir" {
// store the puppet files on disk for other binaries to see and use
prefix := fmt.Sprintf("%s-%s-%s", data.Program, data.Hostname, strings.Replace(PuppetSite, "/", "", -1))
tmpdirName, err := ioutil.TempDir("", prefix)
if err != nil {
return errwrap.Wrapf(err, "can't create temp dir")
}
if tmpdirName == "" || tmpdirName == "/" {
return fmt.Errorf("bad tmpdir created")
}
obj.puppetDir = tmpdirName // path to temp dir
// TODO: this code path is untested! test and then rm this notice
if err := util.CopyFsToDisk(fs, PuppetSite, tmpdirName, false); err != nil {
return errwrap.Wrapf(err, "can't copy dir")
}
}
if fi, err := fs.Stat(PuppetConf); err == nil && !fi.IsDir() { // if exists?
b, err := fs.ReadFile(PuppetConf) // read the single file out of it
if err != nil {
return errwrap.Wrapf(err, "can't read config from file `%s`", PuppetConf)
}
// store the puppet conf on disk for other binaries to see and use
prefix := fmt.Sprintf("%s-%s-%s", data.Program, data.Hostname, strings.Replace(PuppetConf, "/", "", -1))
tmpfile, err := ioutil.TempFile("", prefix)
if err != nil {
return errwrap.Wrapf(err, "can't create temp file")
}
obj.puppetConf = tmpfile.Name() // path to temp file
defer tmpfile.Close()
if _, err := tmpfile.Write(b); err != nil {
return errwrap.Wrapf(err, "can't write file")
}
}
obj.closeChan = make(chan struct{})
obj.initialized = true
return nil
@@ -64,9 +244,9 @@ func (obj *GAPI) Init(data gapi.Data) error {
// Graph returns a current Graph.
func (obj *GAPI) Graph() (*pgraph.Graph, error) {
if !obj.initialized {
return nil, fmt.Errorf("the puppet GAPI is not initialized")
return nil, fmt.Errorf("%s: GAPI is not initialized", Name)
}
config := ParseConfigFromPuppet(*obj.PuppetParam, obj.PuppetConf)
config := obj.ParseConfigFromPuppet()
if config == nil {
return nil, fmt.Errorf("function ParseConfigFromPuppet returned nil")
}
@@ -77,7 +257,7 @@ func (obj *GAPI) Graph() (*pgraph.Graph, error) {
// Next returns nil errors every time there could be a new graph.
func (obj *GAPI) Next() chan gapi.Next {
puppetChan := func() <-chan time.Time { // helper function
return time.Tick(time.Duration(RefreshInterval(obj.PuppetConf)) * time.Second)
return time.Tick(time.Duration(obj.refreshInterval()) * time.Second)
}
ch := make(chan gapi.Next)
obj.wg.Add(1)
@@ -86,7 +266,7 @@ func (obj *GAPI) Next() chan gapi.Next {
defer close(ch) // this will run before the obj.wg.Done()
if !obj.initialized {
next := gapi.Next{
Err: fmt.Errorf("the puppet GAPI is not initialized"),
Err: fmt.Errorf("%s: GAPI is not initialized", Name),
Exit: true, // exit, b/c programming error?
}
ch <- next
@@ -117,7 +297,7 @@ func (obj *GAPI) Next() chan gapi.Next {
return
}
log.Printf("Puppet: Generating new graph...")
log.Printf("%s: Generating new graph...", Name)
if obj.data.NoStreamWatch {
pChan = nil
} else {
@@ -141,8 +321,21 @@ func (obj *GAPI) Next() chan gapi.Next {
// Close shuts down the Puppet GAPI.
func (obj *GAPI) Close() error {
if !obj.initialized {
return fmt.Errorf("the puppet GAPI is not initialized")
return fmt.Errorf("%s: GAPI is not initialized", Name)
}
if obj.puppetFile != "" {
os.Remove(obj.puppetFile) // clean up, don't bother with error
}
// make this as safe as possible, check we're removing a tempdir too!
if obj.puppetDir != "" && obj.puppetDir != "/" && strings.HasPrefix(obj.puppetDir, os.TempDir()) {
os.RemoveAll(obj.puppetDir)
}
obj.puppetString = "" // free!
if obj.puppetConf != "" {
os.Remove(obj.puppetConf)
}
close(obj.closeChan)
obj.wg.Wait()
obj.initialized = false // closed = true

61
puppet/puppet.go
View File

@@ -20,6 +20,7 @@ package puppet
import (
"bufio"
"fmt"
"io"
"log"
"os/exec"
@@ -38,22 +39,22 @@ const (
func runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
if Debug {
log.Printf("Puppet: running command: %v", cmd)
log.Printf("%s: running command: %v", Name, cmd)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Printf("Puppet: Error opening pipe to puppet command: %v", err)
log.Printf("%s: Error opening pipe to puppet command: %v", Name, err)
return nil, err
}
stderr, err := cmd.StderrPipe()
if err != nil {
log.Printf("Puppet: Error opening error pipe to puppet command: %v", err)
log.Printf("%s: Error opening error pipe to puppet command: %v", Name, err)
return nil, err
}
if err := cmd.Start(); err != nil {
log.Printf("Puppet: Error starting puppet command: %v", err)
log.Printf("%s: Error starting puppet command: %v", Name, err)
return nil, err
}
@@ -65,7 +66,7 @@ func runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
var count int
count, err = stdout.Read(data)
if err != nil && err != io.EOF {
log.Printf("Puppet: Error reading YAML data from puppet: %v", err)
log.Printf("%s: Error reading YAML data from puppet: %v", Name, err)
return nil, err
}
// Slicing down to the number of actual bytes is important, the YAML parser
@@ -73,44 +74,50 @@ func runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
result = append(result, data[0:count]...)
}
if Debug {
log.Printf("Puppet: read %v bytes of data from puppet", len(result))
log.Printf("%s: read %d bytes of data from puppet", Name, len(result))
}
for scanner := bufio.NewScanner(stderr); scanner.Scan(); {
log.Printf("Puppet: (output) %v", scanner.Text())
log.Printf("%s: (output) %v", Name, scanner.Text())
}
if err := cmd.Wait(); err != nil {
log.Printf("Puppet: Error: puppet command did not complete: %v", err)
log.Printf("%s: Error: puppet command did not complete: %v", Name, err)
return nil, err
}
return result, nil
}
// ParseConfigFromPuppet takes a special puppet param string and config and
// returns the graph configuration structure.
func ParseConfigFromPuppet(puppetParam, puppetConf string) *yamlgraph.GraphConfig {
// ParseConfigFromPuppet returns the graph configuration structure from the mode
// and input values, including possibly some file and directory paths.
func (obj *GAPI) ParseConfigFromPuppet() *yamlgraph.GraphConfig {
var args []string
if puppetParam == "agent" {
switch obj.Mode {
case "agent":
args = []string{"mgmtgraph", "print"}
} else if strings.HasSuffix(puppetParam, ".pp") {
args = []string{"mgmtgraph", "print", "--manifest", puppetParam}
} else {
args = []string{"mgmtgraph", "print", "--code", puppetParam}
case "file":
args = []string{"mgmtgraph", "print", "--manifest", obj.puppetFile}
case "string":
args = []string{"mgmtgraph", "print", "--code", obj.puppetString}
case "dir":
// TODO: run the code from the obj.puppetDir directory path
return nil // XXX: not implemented
default:
panic(fmt.Sprintf("%s: unhandled case: %s", Name, obj.Mode))
}
if puppetConf != "" {
args = append(args, "--config="+puppetConf)
if obj.puppetConf != "" {
args = append(args, "--config="+obj.puppetConf)
}
cmd := exec.Command("puppet", args...)
log.Println("Puppet: launching translator")
log.Printf("%s: launching translator", Name)
var config yamlgraph.GraphConfig
if data, err := runPuppetCommand(cmd); err != nil {
return nil
} else if err := config.Parse(data); err != nil {
log.Printf("Puppet: Error: Could not parse YAML output with Parse: %v", err)
log.Printf("%s: Error: Could not parse YAML output with Parse: %v", Name, err)
return nil
}
@@ -118,28 +125,28 @@ func ParseConfigFromPuppet(puppetParam, puppetConf string) *yamlgraph.GraphConfi
}
// RefreshInterval returns the graph refresh interval from the puppet configuration.
func RefreshInterval(puppetConf string) int {
func (obj *GAPI) refreshInterval() int {
if Debug {
log.Printf("Puppet: determining graph refresh interval")
log.Printf("%s: determining graph refresh interval", Name)
}
var cmd *exec.Cmd
if puppetConf != "" {
cmd = exec.Command("puppet", "config", "print", "runinterval", "--config", puppetConf)
if obj.puppetConf != "" {
cmd = exec.Command("puppet", "config", "print", "runinterval", "--config", obj.puppetConf)
} else {
cmd = exec.Command("puppet", "config", "print", "runinterval")
}
log.Println("Puppet: inspecting runinterval configuration")
log.Printf("%s: inspecting runinterval configuration", Name)
interval := 1800
data, err := runPuppetCommand(cmd)
if err != nil {
log.Printf("Puppet: could not determine configured run interval (%v), using default of %v", err, interval)
log.Printf("%s: could not determine configured run interval (%v), using default of %v", Name, err, interval)
return interval
}
result, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 0)
if err != nil {
log.Printf("Puppet: error reading numeric runinterval value (%v), using default of %v", err, interval)
log.Printf("%s: error reading numeric runinterval value (%v), using default of %v", Name, err, interval)
return interval
}