From 6ae3481ae933e4f626415768f9bc5bffc63d6aa8 Mon Sep 17 00:00:00 2001 From: James Shubin Date: Wed, 21 Feb 2024 13:27:18 -0500 Subject: [PATCH] engine, lang, gapi: Split out some functions to a writeable API Start breaking down the filesystem interface to make things more flexible. --- engine/fs.go | 18 +++++++++++++++++- gapi/helpers.go | 6 +++--- lang/gapi/gapi.go | 10 ++++++++-- lang/inputs/inputs.go | 12 ++++++------ puppet/gapi.go | 12 +++++++++--- yamlgraph/gapi.go | 8 +++++++- 6 files changed, 50 insertions(+), 16 deletions(-) diff --git a/engine/fs.go b/engine/fs.go index e406d52e..52be1c4d 100644 --- a/engine/fs.go +++ b/engine/fs.go @@ -56,6 +56,22 @@ type Fs interface { TempFile(dir, prefix string) (f afero.File, err error) // slightly different from upstream //UnicodeSanitize(s string) string //Walk(root string, walkFn filepath.WalkFunc) error - WriteFile(filename string, data []byte, perm os.FileMode) error + //WriteFile(filename string, data []byte, perm os.FileMode) error //WriteReader(path string, r io.Reader) (err error) } + +// WriteableFS is our internal filesystem interface for filesystems we write to. +// It can wrap whatever implementations we want. +type WriteableFS interface { + Fs + + // WriteFile writes data to the named file, creating it if necessary. If + // the file does not exist, WriteFile creates it with permissions perm + // (before umask); otherwise WriteFile truncates it before writing, + // without changing permissions. Since Writefile requires multiple + // system calls to complete, a failure mid-operation can leave the file + // in a partially written state. + // + // This mimics the internal os.WriteFile function and has the same docs. + WriteFile(name string, data []byte, perm os.FileMode) error +} diff --git a/gapi/helpers.go b/gapi/helpers.go index fb74078f..1538dd8e 100644 --- a/gapi/helpers.go +++ b/gapi/helpers.go @@ -30,7 +30,7 @@ import ( const Umask = 0666 // CopyFileToFs copies a file from src path on the local fs to a dst path on fs. -func CopyFileToFs(fs engine.Fs, src, dst string) error { +func CopyFileToFs(fs engine.WriteableFS, src, dst string) error { data, err := ioutil.ReadFile(src) if err != nil { return errwrap.Wrapf(err, "can't read from file `%s`", src) @@ -42,7 +42,7 @@ func CopyFileToFs(fs engine.Fs, src, dst string) error { } // CopyBytesToFs copies a list of bytes to a dst path on fs. -func CopyBytesToFs(fs engine.Fs, b []byte, dst string) error { +func CopyBytesToFs(fs engine.WriteableFS, b []byte, dst string) error { if err := fs.WriteFile(dst, b, Umask); err != nil { return errwrap.Wrapf(err, "can't write to file `%s`", dst) } @@ -50,7 +50,7 @@ func CopyBytesToFs(fs engine.Fs, b []byte, dst string) error { } // CopyStringToFs copies a string to a dst path on fs. -func CopyStringToFs(fs engine.Fs, str, dst string) error { +func CopyStringToFs(fs engine.WriteableFS, str, dst string) error { if err := fs.WriteFile(dst, []byte(str), Umask); err != nil { return errwrap.Wrapf(err, "can't write to file `%s`", dst) } diff --git a/lang/gapi/gapi.go b/lang/gapi/gapi.go index fb952a99..3d909fcf 100644 --- a/lang/gapi/gapi.go +++ b/lang/gapi/gapi.go @@ -26,6 +26,7 @@ import ( "sync" "time" + "github.com/purpleidea/mgmt/engine" "github.com/purpleidea/mgmt/gapi" "github.com/purpleidea/mgmt/lang" "github.com/purpleidea/mgmt/lang/ast" @@ -373,9 +374,14 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) { files = append(files, output.Files...) files = append(files, fileList...) + writeableFS, ok := fs.(engine.WriteableFS) + if !ok { + return nil, fmt.Errorf("the FS was not writeable") + } + // run some copy operations to add data into the filesystem for _, fn := range output.Workers { - if err := fn(fs); err != nil { + if err := fn(writeableFS); err != nil { return nil, err } } @@ -437,7 +443,7 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) { continue } // it's a regular file path - if err := gapi.CopyFileToFs(fs, src, dst); err != nil { + if err := gapi.CopyFileToFs(writeableFS, src, dst); err != nil { return nil, errwrap.Wrapf(err, "can't copy file from `%s` to `%s`", src, dst) } } diff --git a/lang/inputs/inputs.go b/lang/inputs/inputs.go index def251f4..8150fbc2 100644 --- a/lang/inputs/inputs.go +++ b/lang/inputs/inputs.go @@ -71,7 +71,7 @@ type ParsedInput struct { Main []byte // contents of main entry mcl code Files []string // files and dirs to copy to fs (abs paths) Metadata *interfaces.Metadata - Workers []func(engine.Fs) error // copy files here that aren't listed! + Workers []func(engine.WriteableFS) error // copy files here that aren't listed! } // ParseInput runs the list if input parsers to know how to run the lexer, @@ -256,8 +256,8 @@ func inputMcl(s string, fs engine.Fs) (*ParsedInput, error) { return nil, errwrap.Wrapf(err, "can't built metadata file") } dst := "/" + interfaces.MetadataFilename // eg: /metadata.yaml - workers := []func(engine.Fs) error{ - func(fs engine.Fs) error { + workers := []func(engine.WriteableFS) error{ + func(fs engine.WriteableFS) error { err := gapi.CopyBytesToFs(fs, byt, dst) return errwrap.Wrapf(err, "could not copy metadata file to fs") }, @@ -354,12 +354,12 @@ func inputCode(s string, fs engine.Fs) (*ParsedInput, error) { dst2 := "/" + metadata.Main // eg: /main.mcl b := []byte(s) // unfortunately we convert things back and forth :/ - workers := []func(engine.Fs) error{ - func(fs engine.Fs) error { + workers := []func(engine.WriteableFS) error{ + func(fs engine.WriteableFS) error { err := gapi.CopyBytesToFs(fs, byt, dst1) return errwrap.Wrapf(err, "could not copy metadata file to fs") }, - func(fs engine.Fs) error { + func(fs engine.WriteableFS) error { err := gapi.CopyBytesToFs(fs, b, dst2) return errwrap.Wrapf(err, "could not copy main file to fs") }, diff --git a/puppet/gapi.go b/puppet/gapi.go index adf280ae..75bfcce8 100644 --- a/puppet/gapi.go +++ b/puppet/gapi.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/purpleidea/mgmt/engine" "github.com/purpleidea/mgmt/gapi" "github.com/purpleidea/mgmt/pgraph" "github.com/purpleidea/mgmt/util" @@ -105,6 +106,11 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) { return nil, fmt.Errorf("%s input is empty", Name) } + writeableFS, ok := fs.(engine.WriteableFS) + if !ok { + return nil, fmt.Errorf("the FS was not writeable") + } + isDir := func(p string) (bool, error) { if !strings.HasPrefix(p, "/") { return false, nil @@ -125,7 +131,7 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) { } else if strings.HasSuffix(s, ".pp") { mode = "file" - if err := gapi.CopyFileToFs(fs, s, PuppetFile); err != nil { + if err := gapi.CopyFileToFs(writeableFS, s, PuppetFile); err != nil { return nil, errwrap.Wrapf(err, "can't copy code from `%s` to `%s`", s, PuppetFile) } @@ -142,14 +148,14 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) { } else { mode = "string" - if err := gapi.CopyStringToFs(fs, s, PuppetFile); err != nil { + if err := gapi.CopyStringToFs(writeableFS, s, PuppetFile); err != nil { return nil, errwrap.Wrapf(err, "can't copy code to `%s`", PuppetFile) } } // TODO: do we want to include this if we have mode == "dir" ? if pc := c.String("puppet-conf"); c.IsSet("puppet-conf") { - if err := gapi.CopyFileToFs(fs, pc, PuppetConf); err != nil { + if err := gapi.CopyFileToFs(writeableFS, pc, PuppetConf); err != nil { return nil, errwrap.Wrapf(err, "can't copy puppet conf from `%s` to '%s'", pc, PuppetConf) } diff --git a/yamlgraph/gapi.go b/yamlgraph/gapi.go index 1a155f98..1a180833 100644 --- a/yamlgraph/gapi.go +++ b/yamlgraph/gapi.go @@ -22,6 +22,7 @@ import ( "fmt" "sync" + "github.com/purpleidea/mgmt/engine" "github.com/purpleidea/mgmt/gapi" "github.com/purpleidea/mgmt/pgraph" "github.com/purpleidea/mgmt/util/errwrap" @@ -86,8 +87,13 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) { return nil, fmt.Errorf("input yaml is empty") } + writeableFS, ok := fs.(engine.WriteableFS) + if !ok { + return nil, fmt.Errorf("the FS was not writeable") + } + // single file input only - if err := gapi.CopyFileToFs(fs, s, Start); err != nil { + if err := gapi.CopyFileToFs(writeableFS, s, Start); err != nil { return nil, errwrap.Wrapf(err, "can't copy yaml from `%s` to `%s`", s, Start) }