engine, lang, gapi: Split out some functions to a writeable API

Start breaking down the filesystem interface to make things more
flexible.

engine/fs.go

@@ -56,6 +56,22 @@ type Fs interface {
 	TempFile(dir, prefix string) (f afero.File, err error) // slightly different from upstream
 	//UnicodeSanitize(s string) string
 	//Walk(root string, walkFn filepath.WalkFunc) error
-	WriteFile(filename string, data []byte, perm os.FileMode) error
+	//WriteFile(filename string, data []byte, perm os.FileMode) error
 	//WriteReader(path string, r io.Reader) (err error)
 }
+
+// WriteableFS is our internal filesystem interface for filesystems we write to.
+// It can wrap whatever implementations we want.
+type WriteableFS interface {
+	Fs
+
+	// WriteFile writes data to the named file, creating it if necessary. If
+	// the file does not exist, WriteFile creates it with permissions perm
+	// (before umask); otherwise WriteFile truncates it before writing,
+	// without changing permissions. Since Writefile requires multiple
+	// system calls to complete, a failure mid-operation can leave the file
+	// in a partially written state.
+	//
+	// This mimics the internal os.WriteFile function and has the same docs.
+	WriteFile(name string, data []byte, perm os.FileMode) error
+}

gapi/helpers.go

@@ -30,7 +30,7 @@ import (
 const Umask = 0666
 
 // CopyFileToFs copies a file from src path on the local fs to a dst path on fs.
-func CopyFileToFs(fs engine.Fs, src, dst string) error {
+func CopyFileToFs(fs engine.WriteableFS, src, dst string) error {
 	data, err := ioutil.ReadFile(src)
 	if err != nil {
 		return errwrap.Wrapf(err, "can't read from file `%s`", src)
@@ -42,7 +42,7 @@ func CopyFileToFs(fs engine.Fs, src, dst string) error {
 }
 
 // CopyBytesToFs copies a list of bytes to a dst path on fs.
-func CopyBytesToFs(fs engine.Fs, b []byte, dst string) error {
+func CopyBytesToFs(fs engine.WriteableFS, b []byte, dst string) error {
 	if err := fs.WriteFile(dst, b, Umask); err != nil {
 		return errwrap.Wrapf(err, "can't write to file `%s`", dst)
 	}
@@ -50,7 +50,7 @@ func CopyBytesToFs(fs engine.Fs, b []byte, dst string) error {
 }
 
 // CopyStringToFs copies a string to a dst path on fs.
-func CopyStringToFs(fs engine.Fs, str, dst string) error {
+func CopyStringToFs(fs engine.WriteableFS, str, dst string) error {
 	if err := fs.WriteFile(dst, []byte(str), Umask); err != nil {
 		return errwrap.Wrapf(err, "can't write to file `%s`", dst)
 	}

lang/gapi/gapi.go

@@ -26,6 +26,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/gapi"
 	"github.com/purpleidea/mgmt/lang"
 	"github.com/purpleidea/mgmt/lang/ast"
@@ -373,9 +374,14 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) {
 	files = append(files, output.Files...)
 	files = append(files, fileList...)
 
+	writeableFS, ok := fs.(engine.WriteableFS)
+	if !ok {
+		return nil, fmt.Errorf("the FS was not writeable")
+	}
+
 	// run some copy operations to add data into the filesystem
 	for _, fn := range output.Workers {
-		if err := fn(fs); err != nil {
+		if err := fn(writeableFS); err != nil {
 			return nil, err
 		}
 	}
@@ -437,7 +443,7 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) {
 			continue
 		}
 		// it's a regular file path
-		if err := gapi.CopyFileToFs(fs, src, dst); err != nil {
+		if err := gapi.CopyFileToFs(writeableFS, src, dst); err != nil {
 			return nil, errwrap.Wrapf(err, "can't copy file from `%s` to `%s`", src, dst)
 		}
 	}

lang/inputs/inputs.go

@@ -71,7 +71,7 @@ type ParsedInput struct {
 	Main     []byte    // contents of main entry mcl code
 	Files    []string  // files and dirs to copy to fs (abs paths)
 	Metadata *interfaces.Metadata
-	Workers  []func(engine.Fs) error // copy files here that aren't listed!
+	Workers  []func(engine.WriteableFS) error // copy files here that aren't listed!
 }
 
 // ParseInput runs the list if input parsers to know how to run the lexer,
@@ -256,8 +256,8 @@ func inputMcl(s string, fs engine.Fs) (*ParsedInput, error) {
 		return nil, errwrap.Wrapf(err, "can't built metadata file")
 	}
 	dst := "/" + interfaces.MetadataFilename // eg: /metadata.yaml
-	workers := []func(engine.Fs) error{
-		func(fs engine.Fs) error {
+	workers := []func(engine.WriteableFS) error{
+		func(fs engine.WriteableFS) error {
 			err := gapi.CopyBytesToFs(fs, byt, dst)
 			return errwrap.Wrapf(err, "could not copy metadata file to fs")
 		},
@@ -354,12 +354,12 @@ func inputCode(s string, fs engine.Fs) (*ParsedInput, error) {
 	dst2 := "/" + metadata.Main               // eg: /main.mcl
 	b := []byte(s)                            // unfortunately we convert things back and forth :/
 
-	workers := []func(engine.Fs) error{
-		func(fs engine.Fs) error {
+	workers := []func(engine.WriteableFS) error{
+		func(fs engine.WriteableFS) error {
 			err := gapi.CopyBytesToFs(fs, byt, dst1)
 			return errwrap.Wrapf(err, "could not copy metadata file to fs")
 		},
-		func(fs engine.Fs) error {
+		func(fs engine.WriteableFS) error {
 			err := gapi.CopyBytesToFs(fs, b, dst2)
 			return errwrap.Wrapf(err, "could not copy main file to fs")
 		},

puppet/gapi.go

@@ -25,6 +25,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/gapi"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
@@ -105,6 +106,11 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) {
 		return nil, fmt.Errorf("%s input is empty", Name)
 	}
 
+	writeableFS, ok := fs.(engine.WriteableFS)
+	if !ok {
+		return nil, fmt.Errorf("the FS was not writeable")
+	}
+
 	isDir := func(p string) (bool, error) {
 		if !strings.HasPrefix(p, "/") {
 			return false, nil
@@ -125,7 +131,7 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) {
 
 	} else if strings.HasSuffix(s, ".pp") {
 		mode = "file"
-		if err := gapi.CopyFileToFs(fs, s, PuppetFile); err != nil {
+		if err := gapi.CopyFileToFs(writeableFS, s, PuppetFile); err != nil {
 			return nil, errwrap.Wrapf(err, "can't copy code from `%s` to `%s`", s, PuppetFile)
 		}
 
@@ -142,14 +148,14 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) {
 
 	} else {
 		mode = "string"
-		if err := gapi.CopyStringToFs(fs, s, PuppetFile); err != nil {
+		if err := gapi.CopyStringToFs(writeableFS, s, PuppetFile); err != nil {
 			return nil, errwrap.Wrapf(err, "can't copy code to `%s`", PuppetFile)
 		}
 	}
 
 	// TODO: do we want to include this if we have mode == "dir" ?
 	if pc := c.String("puppet-conf"); c.IsSet("puppet-conf") {
-		if err := gapi.CopyFileToFs(fs, pc, PuppetConf); err != nil {
+		if err := gapi.CopyFileToFs(writeableFS, pc, PuppetConf); err != nil {
 			return nil, errwrap.Wrapf(err, "can't copy puppet conf from `%s` to '%s'", pc, PuppetConf)
 
 		}

yamlgraph/gapi.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/gapi"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util/errwrap"
@@ -86,8 +87,13 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) {
 		return nil, fmt.Errorf("input yaml is empty")
 	}
 
+	writeableFS, ok := fs.(engine.WriteableFS)
+	if !ok {
+		return nil, fmt.Errorf("the FS was not writeable")
+	}
+
 	// single file input only
-	if err := gapi.CopyFileToFs(fs, s, Start); err != nil {
+	if err := gapi.CopyFileToFs(writeableFS, s, Start); err != nil {
 		return nil, errwrap.Wrapf(err, "can't copy yaml from `%s` to `%s`", s, Start)
 	}