// Mgmt // Copyright (C) James Shubin and the project contributors // Written by James Shubin and the project contributors // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . // // Additional permission under GNU GPL version 3 section 7 // // If you modify this program, or any covered work, by linking or combining it // with embedded mcl code and modules (and that the embedded mcl code and // modules which link with this program, contain a copy of their source code in // the authoritative form) containing parts covered by the terms of any other // license, the licensors of this program grant you additional permission to // convey the resulting work. Furthermore, the licensors of this program grant // the original author, James Shubin, additional permission to update this // additional permission if he deems it necessary to achieve the goals of this // additional permission. package interfaces import ( "fmt" "io" "strings" "github.com/purpleidea/mgmt/util/errwrap" "gopkg.in/yaml.v2" ) const ( // MetadataFilename is the filename for the metadata storage. This is // the ideal entry point for any running code. MetadataFilename = "metadata.yaml" // FileNameExtension is the filename extension used for languages files. FileNameExtension = "mcl" // alternate suggestions welcome! // DotFileNameExtension is the filename extension with a dot prefix. DotFileNameExtension = "." + FileNameExtension // MainFilename is the default filename for code to start running from. MainFilename = "main" + DotFileNameExtension // PathDirectory is the path directory name we search for modules in. PathDirectory = "path/" // FilesDirectory is the files directory name we include alongside // modules. It can store any useful files that we'd like. FilesDirectory = "files/" // ModuleDirectory is the default module directory name. It gets // appended to whatever the running prefix is or relative to the base // dir being used for deploys. ModuleDirectory = "modules/" ) // Metadata is a data structure representing the module metadata. Since it can // get moved around to different filesystems, it should only contain relative // paths. type Metadata struct { // Main is the path to the entry file where we start reading code. // Normally this is main.mcl or the value of the MainFilename constant. Main string `yaml:"main"` // Path is the relative path to the local module search path directory // that we should look in. This is similar to golang's vendor directory. // If a module wishes to include this directory, it's recommended that // it have the contained directory be a `git submodule` if possible. Path string `yaml:"path"` // Files is the location of the files/ directory which can contain some // useful additions that might get used in the modules. You can store // templates, or any other data that you'd like. // TODO: also allow storing files alongside the .mcl files in their dir! Files string `yaml:"files"` // License is the listed license of the module. Use the short names, eg: // LGPLv3+, or MIT. License string `yaml:"license"` // ParentPathBlock specifies whether we're allowed to search in parent // metadata file Path settings for modules. We always search in the // global path if we don't find others first. This setting defaults to // false, which is important because the downloader uses it to decide // where to put downloaded modules. It is similar to the equivalent of // a `require vendoring` flag in golang if such a thing existed. If a // module sets this to true, and specifies a Path value, then only that // path will be used as long as imports are present there. Otherwise it // will fall-back on the global modules directory. If a module sets this // to true, and does not specify a Path value, then the global modules // directory is automatically chosen for the import location for this // module. When this is set to true, in no scenario will an import come // from a directory other than the one specified here, or the global // modules directory. Module authors should use this sparingly when they // absolutely need a specific import vendored, otherwise they might // rouse the ire of module consumers. Keep in mind that you can specify // a Path directory, and include a git submodule in it, which will be // used by default, without specifying this option. In that scenario, // the consumer can decide to not recursively clone your submodule if // they wish to override it higher up in the module search locations. ParentPathBlock bool `yaml:"parentpathblock"` // Metadata stores a link to the parent metadata structure if it exists. Metadata *Metadata // this does *NOT* get a yaml struct tag // metadataPath stores the absolute path to this metadata file as it is // parsed. This is useful when we search upwards for parent Path values. metadataPath string // absolute path that this file was found in // TODO: is this needed anymore? defaultMain *string // set this to pick a default Main when decoding // bug395 is a flag to workaround the terrible yaml parser resetting all // the default struct field values when it finds an empty yaml document. // We set this value to have a default of true, which enables us to know // if the document was empty or not, and if so, then we know this struct // was emptied, so we should then return a new struct with all defaults. // See: https://github.com/go-yaml/yaml/issues/395 for more information. bug395 bool } // DefaultMetadata returns the default metadata that is used for absent values. func DefaultMetadata() *Metadata { return &Metadata{ // the defaults Main: MainFilename, // main.mcl // This MUST be empty for a top-level default, because if it's // not, then an undefined Path dir at a lower level won't search // upwards to find a suitable path, and we'll nest forever... //Path: PathDirectory, // do NOT set this! Files: FilesDirectory, // files/ //License: "", // TODO: ??? bug395: true, // workaround, lol } } // SetAbsSelfPath sets the absolute directory path to this metadata file. This // method is used on a built metadata file so that it can internally know where // it is located. func (obj *Metadata) SetAbsSelfPath(p string) error { obj.metadataPath = p return nil } // ToBytes marshals the struct into a byte array and returns it. func (obj *Metadata) ToBytes() ([]byte, error) { return yaml.Marshal(obj) // TODO: obj or *obj ? } // NOTE: this is not currently needed, but here for reference. //// MarshalYAML modifies the struct before it is used to build the raw output. //func (obj *Metadata) MarshalYAML() (interface{}, error) { // // The Marshaler interface may be implemented by types to customize // // their behavior when being marshaled into a YAML document. The // // returned value is marshaled in place of the original value // // implementing Marshaler. // // if obj.metadataPath == "" { // make sure metadataPath isn't saved! // return obj, nil // } // md := obj.Copy() // TODO: implement me // md.metadataPath = "" // if set, blank it out before save // return md, nil //} // UnmarshalYAML is the standard unmarshal method for this struct. func (obj *Metadata) UnmarshalYAML(unmarshal func(interface{}) error) error { type indirect Metadata // indirection to avoid infinite recursion def := DefaultMetadata() // support overriding if x := obj.defaultMain; x != nil { def.Main = *x } raw := indirect(*def) // convert; the defaults go here if err := unmarshal(&raw); err != nil { return err } *obj = Metadata(raw) // restore from indirection with type conversion! return nil } // ParseMetadata reads from some input and returns a *Metadata struct that // contains plausible values to be used. func ParseMetadata(reader io.Reader) (*Metadata, error) { metadata := DefaultMetadata() // populate this //main := MainFilename // set a custom default here if you want //metadata.defaultMain = &main // does not work in all cases :/ (fails with EOF files, ioutil does not) //decoder := yaml.NewDecoder(reader) ////decoder.SetStrict(true) // TODO: consider being strict? //if err := decoder.Decode(metadata); err != nil { // return nil, errwrap.Wrapf(err, "can't parse metadata") //} b, err := io.ReadAll(reader) if err != nil { return nil, errwrap.Wrapf(err, "can't read metadata") } if err := yaml.Unmarshal(b, metadata); err != nil { return nil, errwrap.Wrapf(err, "can't parse metadata") } if !metadata.bug395 { // workaround, lol // we must have gotten an empty document, so use a new default! metadata = DefaultMetadata() } // FIXME: search for unclean paths containing ../ or similar and error! if strings.HasPrefix(metadata.Main, "/") || strings.HasSuffix(metadata.Main, "/") { return nil, fmt.Errorf("the Main field must be a relative file path") } if metadata.Path != "" && (strings.HasPrefix(metadata.Path, "/") || !strings.HasSuffix(metadata.Path, "/")) { return nil, fmt.Errorf("the Path field must be undefined or be a relative dir path") } if metadata.Files != "" && (strings.HasPrefix(metadata.Files, "/") || !strings.HasSuffix(metadata.Files, "/")) { return nil, fmt.Errorf("the Files field must be undefined or be a relative dir path") } // TODO: add more validation return metadata, nil } // FindModulesPath returns an absolute path to the Path dir where modules can be // found. This can vary, because the current metadata file might not specify a // Path value, meaning we'd have to return the global modules path. // Additionally, we can search upwards for a path if our metadata file allows // this. It searches with respect to the calling base directory, and uses the // ParentPathBlock field to determine if we're allowed to search upwards. It // does logically without doing any filesystem operations. func FindModulesPath(metadata *Metadata, base, modules string) (string, error) { ret := func(s string) (string, error) { // return helper function // don't return an empty string without an error!!! if s == "" { return "", fmt.Errorf("can't find a module path") } return s, nil } m := metadata // start b := base // absolute base path current metadata file is in for m != nil { if m.metadataPath == "" { // a top-level module might be empty! return ret(modules) // so return this, there's nothing else! } if m.metadataPath != b { // these should be the same if no bugs! return "", fmt.Errorf("metadata inconsistency: `%s` != `%s`", m.metadataPath, b) } // does metadata specify where to look ? // search in the module specific space if m.Path != "" { // use this path, since it was specified! if !strings.HasSuffix(m.Path, "/") { return "", fmt.Errorf("metadata inconsistency: path `%s` has no trailing slash", m.Path) } return ret(b + m.Path) // join w/o cleaning trailing slash } // are we allowed to search incrementally upwards? if m.ParentPathBlock { break } // search upwards (search in parent dirs upwards recursively...) m = m.Metadata // might be nil if m != nil { b = m.metadataPath // get new parent path } } // by now we haven't found a metadata path, so we use the global path... return ret(modules) // often comes from an ENV or a default } // FindModulesPathList does what FindModulesPath does, except this function // returns the entirely linear string of possible module locations until it gets // to the root. This can be useful if you'd like to know which possible // locations are valid, so that you can search through them to see if there is // downloaded code available. func FindModulesPathList(metadata *Metadata, base, modules string) ([]string, error) { found := []string{} ret := func(s []string) ([]string, error) { // return helper function // don't return an empty list without an error!!! if s == nil || len(s) == 0 { return nil, fmt.Errorf("can't find any module paths") } return s, nil } m := metadata // start b := base // absolute base path current metadata file is in for m != nil { if m.metadataPath == "" { // a top-level module might be empty! return ret([]string{modules}) // so return this, there's nothing else! } if m.metadataPath != b { // these should be the same if no bugs! return nil, fmt.Errorf("metadata inconsistency: `%s` != `%s`", m.metadataPath, b) } // does metadata specify where to look ? // search in the module specific space if m.Path != "" { // use this path, since it was specified! if !strings.HasSuffix(m.Path, "/") { return nil, fmt.Errorf("metadata inconsistency: path `%s` has no trailing slash", m.Path) } p := b + m.Path // join w/o cleaning trailing slash found = append(found, p) // add to list } // are we allowed to search incrementally upwards? if m.ParentPathBlock { break } // search upwards (search in parent dirs upwards recursively...) m = m.Metadata // might be nil if m != nil { b = m.metadataPath // get new parent path } } // add the global path to everything we've found... found = append(found, modules) // often comes from an ENV or a default return ret(found) }