lang: Add an embedded package for embedded imports

This adds a new "embedded" package which can be used to import
system-like packages that are embedded into the binary.

etcd/world.go

@@ -32,6 +32,7 @@ import (
 	etcdfs "github.com/purpleidea/mgmt/etcd/fs"
 	"github.com/purpleidea/mgmt/etcd/interfaces"
 	"github.com/purpleidea/mgmt/etcd/scheduler"
+	"github.com/purpleidea/mgmt/lang/embedded"
 	"github.com/purpleidea/mgmt/util"
 )
 
@@ -191,6 +192,11 @@ func (obj *World) Fs(uri string) (engine.Fs, error) {
 		return obj.StandaloneFs, nil
 	}
 
+	if u.Scheme == embedded.Scheme {
+		path := strings.TrimPrefix(u.Path, "/") // expect a leading slash
+		return embedded.Lookup(path)            // does not expect a leading slash
+	}
+
 	if u.Scheme != etcdfs.Scheme {
 		return nil, fmt.Errorf("unknown scheme: `%s`", u.Scheme)
 	}

go.mod

@@ -122,6 +122,7 @@ require (
 	github.com/xanzy/ssh-agent v0.3.2 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/yalue/merged_fs v1.3.0 // indirect
 	go.etcd.io/bbolt v1.3.8 // indirect
 	go.etcd.io/etcd/client/v2 v2.305.10 // indirect
 	go.etcd.io/etcd/pkg/v3 v3.5.10 // indirect

go.sum

@@ -807,6 +807,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
+github.com/yalue/merged_fs v1.3.0 h1:qCeh9tMPNy/i8cwDsQTJ5bLr6IRxbs6meakNE5O+wyY=
+github.com/yalue/merged_fs v1.3.0/go.mod h1:WqqchfVYQyclV2tnR7wtRhBddzBvLVR83Cjw9BKQw0M=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

lang/ast/structs.go

@@ -31,6 +31,7 @@ import (
 	"github.com/purpleidea/mgmt/engine"
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/lang/core"
+	"github.com/purpleidea/mgmt/lang/embedded"
 	"github.com/purpleidea/mgmt/lang/funcs"
 	"github.com/purpleidea/mgmt/lang/funcs/structs"
 	"github.com/purpleidea/mgmt/lang/inputs"
@@ -3235,6 +3236,37 @@ func (obj *StmtProg) importScope(info *interfaces.ImportData, scope *interfaces.
 	// obj.data.Base + obj.data.Metadata.Main
 	// but recursive imports mean this is not always the active file...
 
+	// attempt to load an embedded system import first (pure mcl rather than golang)
+	if fs, err := embedded.Lookup(info.Name); info.IsSystem && err == nil {
+		nextVertex, err := obj.nextVertex(info)
+		if err != nil {
+			return nil, err
+		}
+
+		//tree, err := util.FsTree(fs, "/")
+		//if err != nil {
+		//	return nil, err
+		//}
+		//obj.data.Logf("tree:\n%s", tree)
+
+		s := "/" + interfaces.MetadataFilename // standard entry point
+		//s := "/" // would this directory parser approach be better?
+		input, err := inputs.ParseInput(s, fs) // use my FS
+		if err != nil {
+			return nil, errwrap.Wrapf(err, "embedded could not activate an input parser")
+		}
+
+		// The files we're pulling in are already embedded, so we must
+		// not try to copy them in from disk or it won't succeed.
+		input.Files = []string{} // clear
+
+		embeddedScope, err := obj.importScopeWithParsedInputs(input, scope, nextVertex)
+		if err != nil {
+			return nil, errwrap.Wrapf(err, "embedded import of `%s` failed", info.Name)
+		}
+		return embeddedScope, nil
+	}
+
 	if info.IsSystem { // system imports are the exact name, eg "fmt"
 		systemScope, err := obj.importSystemScope(info.Name)
 		if err != nil {

lang/embedded/embedded.go

@@ -0,0 +1,123 @@
+// Mgmt
+// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package embedded embeds mcl modules into the system import namespace.
+// Typically these are made available via an `import "embedded/foo"` style stmt.
+package embedded
+
+import (
+	"fmt"
+	"io/fs"
+	"runtime"
+	"strings"
+
+	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/util"
+
+	"github.com/spf13/afero"
+	"github.com/yalue/merged_fs"
+)
+
+const (
+	// Scheme is the string used to represent the scheme used by the
+	// embedded filesystem URI.
+	Scheme = "embeddedfs"
+)
+
+var registeredEmbeds = make(map[string]fs.ReadFileFS) // must initialize
+
+// ModuleRegister takes a filesystem and stores a reference to it in our
+// embedded module system along with a name. Future lookups to that name will
+// pull out that filesystem.
+func ModuleRegister(module string, fs fs.ReadFileFS) {
+	if _, exists := registeredEmbeds[module]; exists {
+		panic(fmt.Sprintf("an embed in module %s is already registered", module))
+	}
+
+	// we currently set the fs URI when we return an fs with the Lookup func
+	registeredEmbeds[module] = fs
+}
+
+// Lookup pulls out an embedded filesystem module which will contain a valid URI
+// method. The returned fs is read-only.
+// XXX: Update the interface to remove the afero part leaving this all read-only
+func Lookup(module string) (engine.Fs, error) {
+	fs, exists := registeredEmbeds[module]
+	if !exists {
+		return nil, fmt.Errorf("could not lookup embedded module: %s", module)
+	}
+
+	// XXX: All this horrible filesystem transformation mess happens because
+	// golang doesn't have a writeable io/fs.WriteableFS interface... We can
+	// eventually port this further away from Afero though...
+	fromIOFS := afero.FromIOFS{FS: fs}     // fulfills afero.Fs interface
+	rp := util.NewRelPathFs(fromIOFS, "/") // calls to `/foo` turn into `foo`
+	afs := &afero.Afero{Fs: rp}            // wrap so that we're implementing ioutil
+	engineFS := &util.AferoFs{             // fulfills engine.Fs interface
+		Scheme: Scheme,       // pick the scheme!
+		Path:   "/" + module, // need a leading slash
+		Afero:  afs,
+	}
+	return engineFS, nil
+}
+
+// MergeFS merges multiple filesystems and returns an fs.FS. It is provided as a
+// helper function to abstract away the underlying implementation in case we
+// ever wish to replace it with something more performant or ergonomic.
+// TODO: add a new interface that combines ReadFileFS and ReadDirFS and use that
+// as the signature everywhere so we could catch those issues at the very start!
+func MergeFS(filesystems ...fs.ReadFileFS) fs.ReadFileFS {
+	l := []fs.FS{}
+	for _, x := range filesystems {
+		f, ok := x.(fs.FS)
+		if !ok {
+			// programming error
+			panic("fs does not support basic FS")
+		}
+		l = append(l, f)
+	}
+	// runs NewMergedFS(a, b) in a balanced way recursively
+	ret, ok := merged_fs.MergeMultiple(l...).(fs.ReadFileFS)
+	if !ok {
+		// programming error
+		panic("fs does not support ReadFileFS")
+	}
+	return ret
+}
+
+// FullModuleName is a helper function that returns the embedded module name.
+// This is the parent directory that an embedded code base should use as prefix.
+func FullModuleName(moduleName string) string {
+	pc, _, _, ok := runtime.Caller(1)
+	if !ok {
+		panic("caller info not found")
+	}
+	s := runtime.FuncForPC(pc).Name()
+	chunks := strings.Split(s, "/")
+	if len(chunks) < 2 {
+		// programming error
+		panic("split pattern not found")
+	}
+	name := chunks[len(chunks)-2]
+	if name == "" {
+		panic("name not found")
+	}
+	if moduleName == "" { // in case we only want to know the module name
+		return name
+	}
+	return name + "/" + moduleName // do the concat for the user!
+}

lang/gapi/gapi.go

@@ -419,6 +419,14 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) {
 	util.PathSlice(files).Sort() // sort it
 	for _, src := range files {  // absolute paths
 		// rebase path src to root file system of "/" for etcdfs...
+
+		// everywhere we expect absolute, but we should use relative :/
+		//tree, err := util.FsTree(fs, "/")
+		//if err != nil {
+		//	return nil, err
+		//}
+		//logf("tree:\n%s", tree)
+
 		dst, err := util.Rebase(src, output.Base, "/")
 		if err != nil {
 			// possible programming error

lib/main.go

@@ -486,6 +486,9 @@ func (obj *Main) Run() error {
 	}).Init()
 
 	// implementation of the World API (alternatives can be substituted in)
+	// XXX: The "implementation of the World API" should have more than just
+	// etcd in it, so this could live elsewhere package wise and just have
+	// an etcd component from the etcd package added in.
 	world := &etcd.World{
 		Hostname:       hostname,
 		Client:         etcdClient,