From 70b5ed7067d35d15f91e1793e578617d2b2c15d7 Mon Sep 17 00:00:00 2001 From: James Shubin Date: Thu, 22 Feb 2024 19:02:33 -0500 Subject: [PATCH] lang: Add an embedded package for embedded imports This adds a new "embedded" package which can be used to import system-like packages that are embedded into the binary. --- etcd/world.go | 6 ++ go.mod | 1 + go.sum | 2 + lang/ast/structs.go | 32 ++++++++++ lang/embedded/embedded.go | 123 ++++++++++++++++++++++++++++++++++++++ lang/gapi/gapi.go | 8 +++ lib/main.go | 3 + 7 files changed, 175 insertions(+) create mode 100644 lang/embedded/embedded.go diff --git a/etcd/world.go b/etcd/world.go index 6427c3c4..193f2ed9 100644 --- a/etcd/world.go +++ b/etcd/world.go @@ -32,6 +32,7 @@ import ( etcdfs "github.com/purpleidea/mgmt/etcd/fs" "github.com/purpleidea/mgmt/etcd/interfaces" "github.com/purpleidea/mgmt/etcd/scheduler" + "github.com/purpleidea/mgmt/lang/embedded" "github.com/purpleidea/mgmt/util" ) @@ -191,6 +192,11 @@ func (obj *World) Fs(uri string) (engine.Fs, error) { return obj.StandaloneFs, nil } + if u.Scheme == embedded.Scheme { + path := strings.TrimPrefix(u.Path, "/") // expect a leading slash + return embedded.Lookup(path) // does not expect a leading slash + } + if u.Scheme != etcdfs.Scheme { return nil, fmt.Errorf("unknown scheme: `%s`", u.Scheme) } diff --git a/go.mod b/go.mod index 5bd57fc0..c8413011 100644 --- a/go.mod +++ b/go.mod @@ -122,6 +122,7 @@ require ( github.com/xanzy/ssh-agent v0.3.2 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + github.com/yalue/merged_fs v1.3.0 // indirect go.etcd.io/bbolt v1.3.8 // indirect go.etcd.io/etcd/client/v2 v2.305.10 // indirect go.etcd.io/etcd/pkg/v3 v3.5.10 // indirect diff --git a/go.sum b/go.sum index 77e0be54..987a002c 100644 --- a/go.sum +++ b/go.sum @@ -807,6 +807,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yalue/merged_fs v1.3.0 h1:qCeh9tMPNy/i8cwDsQTJ5bLr6IRxbs6meakNE5O+wyY= +github.com/yalue/merged_fs v1.3.0/go.mod h1:WqqchfVYQyclV2tnR7wtRhBddzBvLVR83Cjw9BKQw0M= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/lang/ast/structs.go b/lang/ast/structs.go index e2e95d8f..854e44ae 100644 --- a/lang/ast/structs.go +++ b/lang/ast/structs.go @@ -31,6 +31,7 @@ import ( "github.com/purpleidea/mgmt/engine" engineUtil "github.com/purpleidea/mgmt/engine/util" "github.com/purpleidea/mgmt/lang/core" + "github.com/purpleidea/mgmt/lang/embedded" "github.com/purpleidea/mgmt/lang/funcs" "github.com/purpleidea/mgmt/lang/funcs/structs" "github.com/purpleidea/mgmt/lang/inputs" @@ -3235,6 +3236,37 @@ func (obj *StmtProg) importScope(info *interfaces.ImportData, scope *interfaces. // obj.data.Base + obj.data.Metadata.Main // but recursive imports mean this is not always the active file... + // attempt to load an embedded system import first (pure mcl rather than golang) + if fs, err := embedded.Lookup(info.Name); info.IsSystem && err == nil { + nextVertex, err := obj.nextVertex(info) + if err != nil { + return nil, err + } + + //tree, err := util.FsTree(fs, "/") + //if err != nil { + // return nil, err + //} + //obj.data.Logf("tree:\n%s", tree) + + s := "/" + interfaces.MetadataFilename // standard entry point + //s := "/" // would this directory parser approach be better? + input, err := inputs.ParseInput(s, fs) // use my FS + if err != nil { + return nil, errwrap.Wrapf(err, "embedded could not activate an input parser") + } + + // The files we're pulling in are already embedded, so we must + // not try to copy them in from disk or it won't succeed. + input.Files = []string{} // clear + + embeddedScope, err := obj.importScopeWithParsedInputs(input, scope, nextVertex) + if err != nil { + return nil, errwrap.Wrapf(err, "embedded import of `%s` failed", info.Name) + } + return embeddedScope, nil + } + if info.IsSystem { // system imports are the exact name, eg "fmt" systemScope, err := obj.importSystemScope(info.Name) if err != nil { diff --git a/lang/embedded/embedded.go b/lang/embedded/embedded.go new file mode 100644 index 00000000..9445fbbd --- /dev/null +++ b/lang/embedded/embedded.go @@ -0,0 +1,123 @@ +// Mgmt +// Copyright (C) 2013-2024+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +// Package embedded embeds mcl modules into the system import namespace. +// Typically these are made available via an `import "embedded/foo"` style stmt. +package embedded + +import ( + "fmt" + "io/fs" + "runtime" + "strings" + + "github.com/purpleidea/mgmt/engine" + "github.com/purpleidea/mgmt/util" + + "github.com/spf13/afero" + "github.com/yalue/merged_fs" +) + +const ( + // Scheme is the string used to represent the scheme used by the + // embedded filesystem URI. + Scheme = "embeddedfs" +) + +var registeredEmbeds = make(map[string]fs.ReadFileFS) // must initialize + +// ModuleRegister takes a filesystem and stores a reference to it in our +// embedded module system along with a name. Future lookups to that name will +// pull out that filesystem. +func ModuleRegister(module string, fs fs.ReadFileFS) { + if _, exists := registeredEmbeds[module]; exists { + panic(fmt.Sprintf("an embed in module %s is already registered", module)) + } + + // we currently set the fs URI when we return an fs with the Lookup func + registeredEmbeds[module] = fs +} + +// Lookup pulls out an embedded filesystem module which will contain a valid URI +// method. The returned fs is read-only. +// XXX: Update the interface to remove the afero part leaving this all read-only +func Lookup(module string) (engine.Fs, error) { + fs, exists := registeredEmbeds[module] + if !exists { + return nil, fmt.Errorf("could not lookup embedded module: %s", module) + } + + // XXX: All this horrible filesystem transformation mess happens because + // golang doesn't have a writeable io/fs.WriteableFS interface... We can + // eventually port this further away from Afero though... + fromIOFS := afero.FromIOFS{FS: fs} // fulfills afero.Fs interface + rp := util.NewRelPathFs(fromIOFS, "/") // calls to `/foo` turn into `foo` + afs := &afero.Afero{Fs: rp} // wrap so that we're implementing ioutil + engineFS := &util.AferoFs{ // fulfills engine.Fs interface + Scheme: Scheme, // pick the scheme! + Path: "/" + module, // need a leading slash + Afero: afs, + } + return engineFS, nil +} + +// MergeFS merges multiple filesystems and returns an fs.FS. It is provided as a +// helper function to abstract away the underlying implementation in case we +// ever wish to replace it with something more performant or ergonomic. +// TODO: add a new interface that combines ReadFileFS and ReadDirFS and use that +// as the signature everywhere so we could catch those issues at the very start! +func MergeFS(filesystems ...fs.ReadFileFS) fs.ReadFileFS { + l := []fs.FS{} + for _, x := range filesystems { + f, ok := x.(fs.FS) + if !ok { + // programming error + panic("fs does not support basic FS") + } + l = append(l, f) + } + // runs NewMergedFS(a, b) in a balanced way recursively + ret, ok := merged_fs.MergeMultiple(l...).(fs.ReadFileFS) + if !ok { + // programming error + panic("fs does not support ReadFileFS") + } + return ret +} + +// FullModuleName is a helper function that returns the embedded module name. +// This is the parent directory that an embedded code base should use as prefix. +func FullModuleName(moduleName string) string { + pc, _, _, ok := runtime.Caller(1) + if !ok { + panic("caller info not found") + } + s := runtime.FuncForPC(pc).Name() + chunks := strings.Split(s, "/") + if len(chunks) < 2 { + // programming error + panic("split pattern not found") + } + name := chunks[len(chunks)-2] + if name == "" { + panic("name not found") + } + if moduleName == "" { // in case we only want to know the module name + return name + } + return name + "/" + moduleName // do the concat for the user! +} diff --git a/lang/gapi/gapi.go b/lang/gapi/gapi.go index 197d8314..64a0befa 100644 --- a/lang/gapi/gapi.go +++ b/lang/gapi/gapi.go @@ -419,6 +419,14 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) { util.PathSlice(files).Sort() // sort it for _, src := range files { // absolute paths // rebase path src to root file system of "/" for etcdfs... + + // everywhere we expect absolute, but we should use relative :/ + //tree, err := util.FsTree(fs, "/") + //if err != nil { + // return nil, err + //} + //logf("tree:\n%s", tree) + dst, err := util.Rebase(src, output.Base, "/") if err != nil { // possible programming error diff --git a/lib/main.go b/lib/main.go index 11ca8dcb..7776b48f 100644 --- a/lib/main.go +++ b/lib/main.go @@ -486,6 +486,9 @@ func (obj *Main) Run() error { }).Init() // implementation of the World API (alternatives can be substituted in) + // XXX: The "implementation of the World API" should have more than just + // etcd in it, so this could live elsewhere package wise and just have + // an etcd component from the etcd package added in. world := &etcd.World{ Hostname: hostname, Client: etcdClient,