cli, docs: Add a docs command for doc generation

This took a lot longer than it looks to get right. It's not perfect, but
it now reliably generates documentation which we can put into gohugo.

cli/cli.go

@@ -123,6 +123,8 @@ type Args struct {
 
 	FirstbootCmd *FirstbootArgs `arg:"subcommand:firstboot" help:"run some tasks on first boot"`
 
+	DocsCmd *DocsGenerateArgs `arg:"subcommand:docs" help:"generate documentation"`
+
 	// This never runs, it gets preempted in the real main() function.
 	// XXX: Can we do it nicely with the new arg parser? can it ignore all args?
 	EtcdCmd *EtcdArgs `arg:"subcommand:etcd" help:"run standalone etcd"`
@@ -167,6 +169,10 @@ func (obj *Args) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
 		return cmd.Run(ctx, data)
 	}
 
+	if cmd := obj.DocsCmd; cmd != nil {
+		return cmd.Run(ctx, data)
+	}
+
 	// NOTE: we could return true, fmt.Errorf("...") if more than one did
 	return false, nil // nobody activated
 }

cli/docs.go

@@ -0,0 +1,150 @@
+// Mgmt
+// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package cli
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	cliUtil "github.com/purpleidea/mgmt/cli/util"
+	"github.com/purpleidea/mgmt/docs"
+)
+
+// DocsGenerateArgs is the CLI parsing structure and type of the parsed result.
+// This particular one contains all the common flags for the `docs generate`
+// subcommand.
+type DocsGenerateArgs struct {
+	docs.Config // embedded config (can't be a pointer) https://github.com/alexflint/go-arg/issues/240
+
+	DocsGenerate *cliUtil.DocsGenerateArgs `arg:"subcommand:generate" help:"generate documentation"`
+}
+
+// Run executes the correct subcommand. It errors if there's ever an error. It
+// returns true if we did activate one of the subcommands. It returns false if
+// we did not. This information is used so that the top-level parser can return
+// usage or help information if no subcommand activates. This particular Run is
+// the run for the main `docs` subcommand.
+func (obj *DocsGenerateArgs) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	var name string
+	var args interface{}
+	if cmd := obj.DocsGenerate; cmd != nil {
+		name = cliUtil.LookupSubcommand(obj, cmd) // "generate"
+		args = cmd
+	}
+	_ = name
+
+	Logf := func(format string, v ...interface{}) {
+		// Don't block this globally...
+		//if !data.Flags.Debug {
+		//	return
+		//}
+		data.Flags.Logf("main: "+format, v...)
+	}
+
+	var api docs.API
+
+	if cmd := obj.DocsGenerate; cmd != nil {
+		api = &docs.Generate{
+			DocsGenerateArgs: args.(*cliUtil.DocsGenerateArgs),
+			Config:           obj.Config,
+			Program:          data.Program,
+			Version:          data.Version,
+			Debug:            data.Flags.Debug,
+			Logf:             Logf,
+		}
+	}
+
+	if api == nil {
+		return false, nil // nothing found (display help!)
+	}
+
+	// We don't use these for the setup command in normal operation.
+	if data.Flags.Debug {
+		cliUtil.Hello(data.Program, data.Version, data.Flags) // say hello!
+		defer Logf("goodbye!")
+	}
+
+	// install the exit signal handler
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+	exit := make(chan struct{})
+	defer close(exit)
+	wg.Add(1)
+	go func() {
+		defer cancel()
+		defer wg.Done()
+		// must have buffer for max number of signals
+		signals := make(chan os.Signal, 3+1) // 3 * ^C + 1 * SIGTERM
+		signal.Notify(signals, os.Interrupt) // catch ^C
+		//signal.Notify(signals, os.Kill) // catch signals
+		signal.Notify(signals, syscall.SIGTERM)
+		var count uint8
+		for {
+			select {
+			case sig := <-signals: // any signal will do
+				if sig != os.Interrupt {
+					data.Flags.Logf("interrupted by signal")
+					return
+				}
+
+				switch count {
+				case 0:
+					data.Flags.Logf("interrupted by ^C")
+					cancel()
+				case 1:
+					data.Flags.Logf("interrupted by ^C (fast pause)")
+					cancel()
+				case 2:
+					data.Flags.Logf("interrupted by ^C (hard interrupt)")
+					cancel()
+				}
+				count++
+
+			case <-exit:
+				return
+			}
+		}
+	}()
+
+	if err := api.Main(ctx); err != nil {
+		if data.Flags.Debug {
+			data.Flags.Logf("main: %+v", err)
+		}
+		return false, err
+	}
+
+	return true, nil
+}

cli/util/args.go

@@ -187,3 +187,12 @@ type FirstbootStartArgs struct {
 	DoneDir      string `arg:"--done-dir" help:"dir to move done scripts to"`
 	LoggingDir   string `arg:"--logging-dir" help:"directory to store logs in"`
 }
+
+// DocsGenerateArgs is the docgen utility CLI parsing structure and type of the
+// parsed result.
+type DocsGenerateArgs struct {
+	Output      string `arg:"--output" help:"output path to write to"`
+	RootDir     string `arg:"--root-dir" help:"path to mgmt source dir"`
+	NoResources bool   `arg:"--no-resources" help:"skip resource doc generation"`
+	NoFunctions bool   `arg:"--no-functions" help:"skip function doc generation"`
+}

docs/docs.go

@@ -0,0 +1,50 @@
+// Mgmt
+// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+// Package docs provides a tool that generates documentation from the source.
+//
+// ./mgmt docs generate --output /tmp/docs.json && cat /tmp/docs.json | jq
+package docs
+
+import (
+	"context"
+)
+
+// API is the simple interface we expect for any setup items.
+type API interface {
+	// Main runs everything for this setup item.
+	Main(context.Context) error
+}
+
+// Config is a struct of all the configuration values which are shared by all of
+// the setup utilities. By including this as a separate struct, it can be used
+// as part of the API if we want.
+type Config struct {
+	//Foo string `arg:"--foo,env:MGMT_DOCGEN_FOO" help:"Foo..."` // TODO: foo
+}

docs/generate.go

@@ -0,0 +1,788 @@
+// Mgmt
+// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package docs
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	cliUtil "github.com/purpleidea/mgmt/cli/util"
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
+	"github.com/purpleidea/mgmt/engine"
+	engineUtil "github.com/purpleidea/mgmt/engine/util"
+	"github.com/purpleidea/mgmt/lang/funcs"
+	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/util"
+)
+
+const (
+	// JSONSuffix is the output extension for the generated documentation.
+	JSONSuffix = ".json"
+)
+
+// Generate is the main entrypoint for this command. It generates everything.
+type Generate struct {
+	*cliUtil.DocsGenerateArgs // embedded config
+	Config                    // embedded Config
+
+	// Program is the name of this program, usually set at compile time.
+	Program string
+
+	// Version is the version of this program, usually set at compile time.
+	Version string
+
+	// Debug represents if we're running in debug mode or not.
+	Debug bool
+
+	// Logf is a logger which should be used.
+	Logf func(format string, v ...interface{})
+}
+
+// Main runs everything for this setup item.
+func (obj *Generate) Main(ctx context.Context) error {
+	if err := obj.Validate(); err != nil {
+		return err
+	}
+
+	if err := obj.Run(ctx); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Validate verifies that the structure has acceptable data stored within.
+func (obj *Generate) Validate() error {
+	if obj == nil {
+		return fmt.Errorf("data is nil")
+	}
+	if obj.Program == "" {
+		return fmt.Errorf("program is empty")
+	}
+	if obj.Version == "" {
+		return fmt.Errorf("version is empty")
+	}
+
+	return nil
+}
+
+// Run performs the desired actions to generate the documentation.
+func (obj *Generate) Run(ctx context.Context) error {
+
+	outputFile := obj.DocsGenerateArgs.Output
+	if outputFile == "" || !strings.HasSuffix(outputFile, JSONSuffix) {
+		return fmt.Errorf("must specify output")
+	}
+	// support relative paths too!
+	if !strings.HasPrefix(outputFile, "/") {
+		wd, err := os.Getwd()
+		if err != nil {
+			return err
+		}
+		outputFile = filepath.Join(wd, outputFile)
+	}
+
+	if obj.Debug {
+		obj.Logf("output: %s", outputFile)
+	}
+
+	// Ensure the directory exists.
+	//d := filepath.Dir(outputFile)
+	//if err := os.MkdirAll(d, 0750); err != nil {
+	//	return fmt.Errorf("could not make output dir at: %s", d)
+	//}
+
+	resources, err := obj.genResources()
+	if err != nil {
+		return err
+	}
+
+	functions, err := obj.genFunctions()
+	if err != nil {
+		return err
+	}
+
+	data := &Output{
+		Version:   safeVersion(obj.Version),
+		Resources: resources,
+		Functions: functions,
+	}
+
+	b, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+	b = append(b, '\n') // needs a trailing newline
+
+	if err := os.WriteFile(outputFile, b, 0600); err != nil {
+		return err
+	}
+	obj.Logf("wrote: %s", outputFile)
+
+	return nil
+}
+
+func (obj *Generate) getResourceInfo(kind, filename, structName string) (*ResourceInfo, error) {
+	rootDir := obj.DocsGenerateArgs.RootDir
+	if rootDir == "" {
+		wd, err := os.Getwd()
+		if err != nil {
+			return nil, err
+		}
+		rootDir = wd + "/" // add a trailing slash
+	}
+	if !strings.HasPrefix(rootDir, "/") || !strings.HasSuffix(rootDir, "/") {
+		return nil, fmt.Errorf("bad root dir: %s", rootDir)
+	}
+
+	// filename might be "noop.go" for example
+	p := filepath.Join(rootDir, engine.ResourcesRelDir, filename)
+
+	fset := token.NewFileSet()
+
+	// f is a: https://golang.org/pkg/go/ast/#File
+	f, err := parser.ParseFile(fset, p, nil, parser.ParseComments)
+	if err != nil {
+		return nil, err
+	}
+
+	// mcl field name to golang field name
+	mapping, err := engineUtil.LangFieldNameToStructFieldName(kind)
+	if err != nil {
+		return nil, err
+	}
+	// golang field name to mcl field name
+	nameMap, err := util.MapSwap(mapping)
+	if err != nil {
+		return nil, err
+	}
+	// mcl field name to mcl type
+	typMap, err := engineUtil.LangFieldNameToStructType(kind)
+	if err != nil {
+		return nil, err
+	}
+
+	ri := &ResourceInfo{}
+	// Populate the fields, even if they don't have a comment.
+	ri.Name = structName // golang name
+	ri.Kind = kind       // duplicate data
+	ri.File = filename
+	ri.Fields = make(map[string]*ResourceFieldInfo)
+	for mclFieldName, fieldName := range mapping {
+		typ, exists := typMap[mclFieldName]
+		if !exists {
+			continue
+		}
+
+		ri.Fields[mclFieldName] = &ResourceFieldInfo{
+			Name: fieldName,
+			Type: typ.String(),
+			Desc: "", // empty for now
+		}
+	}
+
+	var previousComment *ast.CommentGroup
+
+	// Walk through the AST...
+	ast.Inspect(f, func(node ast.Node) bool {
+
+		// Comments above the struct appear as a node right _before_ we
+		// find the struct, so if we see one, save it for later...
+		if cg, ok := node.(*ast.CommentGroup); ok {
+			previousComment = cg
+			return true
+		}
+
+		typeSpec, ok := node.(*ast.TypeSpec)
+		if !ok {
+			return true
+		}
+		name := typeSpec.Name.Name // name is now known!
+
+		// If the struct isn't what we're expecting, then move on...
+		if name != structName {
+			return true
+		}
+
+		// Check if the TypeSpec is a named struct type that we want...
+		st, ok := typeSpec.Type.(*ast.StructType)
+		if !ok {
+			return true
+		}
+
+		// At this point, we have the struct we want...
+
+		var comment *ast.CommentGroup
+		if typeSpec.Doc != nil {
+			// I don't know how to even get here...
+			comment = typeSpec.Doc // found!
+
+		} else if previousComment != nil {
+			comment = previousComment // found!
+			previousComment = nil
+		}
+
+		ri.Desc = commentCleaner(comment)
+
+		// Iterate over the fields of the struct
+		for _, field := range st.Fields.List {
+			// Check if the field has a comment associated with it
+			if field.Doc == nil {
+				continue
+			}
+
+			if len(field.Names) < 1 { // XXX: why does this happen?
+				continue
+			}
+
+			fieldName := field.Names[0].Name
+			if fieldName == "" { // Can this happen?
+				continue
+			}
+			if isPrivate(fieldName) {
+				continue
+			}
+
+			mclFieldName, exists := nameMap[fieldName]
+			if !exists {
+				continue
+			}
+
+			ri.Fields[mclFieldName].Desc = commentCleaner(field.Doc)
+		}
+
+		return true
+	})
+
+	return ri, nil
+}
+
+func (obj *Generate) genResources() (map[string]*ResourceInfo, error) {
+	resources := make(map[string]*ResourceInfo)
+	if obj.DocsGenerateArgs.NoResources {
+		return resources, nil
+	}
+
+	r := engine.RegisteredResourcesNames()
+	sort.Strings(r)
+	for _, kind := range r {
+		metadata, err := docsUtil.LookupResource(kind)
+		if err != nil {
+			return nil, err
+		}
+
+		if strings.HasPrefix(kind, "_") {
+			// TODO: Should we display these somehow?
+			// built-in resource
+			continue
+		}
+
+		ri, err := obj.getResourceInfo(kind, metadata.Filename, metadata.Typename)
+		if err != nil {
+			return nil, err
+		}
+
+		if ri.Name == "" {
+			return nil, fmt.Errorf("empty resource name: %s", kind)
+		}
+		if ri.File == "" {
+			return nil, fmt.Errorf("empty resource file: %s", kind)
+		}
+		if ri.Desc == "" {
+			obj.Logf("empty resource desc: %s", kind)
+		}
+		fields := []string{}
+		for field := range ri.Fields {
+			fields = append(fields, field)
+		}
+		sort.Strings(fields)
+		for _, field := range fields {
+			if ri.Fields[field].Desc == "" {
+				obj.Logf("empty resource (%s) field desc: %s", kind, field)
+			}
+		}
+
+		resources[kind] = ri
+	}
+
+	return resources, nil
+}
+
+func (obj *Generate) getFunctionInfo(pkg, name string, metadata *docsUtil.Metadata) (*FunctionInfo, error) {
+	rootDir := obj.DocsGenerateArgs.RootDir
+	if rootDir == "" {
+		wd, err := os.Getwd()
+		if err != nil {
+			return nil, err
+		}
+		rootDir = wd + "/" // add a trailing slash
+	}
+	if !strings.HasPrefix(rootDir, "/") || !strings.HasSuffix(rootDir, "/") {
+		return nil, fmt.Errorf("bad root dir: %s", rootDir)
+	}
+	if metadata.Filename == "" {
+		return nil, fmt.Errorf("empty filename for: %s.%s", pkg, name)
+	}
+
+	// filename might be "pow.go" for example and contain a rel dir
+	p := filepath.Join(rootDir, funcs.FunctionsRelDir, metadata.Filename)
+
+	fset := token.NewFileSet()
+
+	// f is a: https://golang.org/pkg/go/ast/#File
+	f, err := parser.ParseFile(fset, p, nil, parser.ParseComments)
+	if err != nil {
+		return nil, err
+	}
+
+	fi := &FunctionInfo{}
+	fi.Name = metadata.Typename
+	fi.File = metadata.Filename
+
+	var previousComment *ast.CommentGroup
+	found := false
+
+	rawFunc := func(node ast.Node) (*ast.CommentGroup, string) {
+		fd, ok := node.(*ast.FuncDecl)
+		if !ok {
+			return nil, ""
+		}
+		return fd.Doc, fd.Name.Name // name is now known!
+	}
+
+	rawStruct := func(node ast.Node) (*ast.CommentGroup, string) {
+		typeSpec, ok := node.(*ast.TypeSpec)
+		if !ok {
+			return nil, ""
+		}
+
+		// Check if the TypeSpec is a named struct type that we want...
+		if _, ok := typeSpec.Type.(*ast.StructType); !ok {
+			return nil, ""
+		}
+
+		return typeSpec.Doc, typeSpec.Name.Name // name is now known!
+	}
+
+	// Walk through the AST...
+	ast.Inspect(f, func(node ast.Node) bool {
+
+		// Comments above the struct appear as a node right _before_ we
+		// find the struct, so if we see one, save it for later...
+		if cg, ok := node.(*ast.CommentGroup); ok {
+			previousComment = cg
+			return true
+		}
+
+		doc, name := rawFunc(node) // First see if it's a raw func.
+		if name == "" {
+			doc, name = rawStruct(node) // Otherwise it's a struct.
+		}
+
+		// If the func isn't what we're expecting, then move on...
+		if name != metadata.Typename {
+			return true
+		}
+
+		var comment *ast.CommentGroup
+		if doc != nil {
+			// I don't know how to even get here...
+			comment = doc // found!
+
+		} else if previousComment != nil {
+			comment = previousComment // found!
+			previousComment = nil
+		}
+
+		fi.Desc = commentCleaner(comment)
+		found = true
+
+		return true
+	})
+
+	if !found {
+		//return nil, nil
+	}
+
+	return fi, nil
+}
+
+func (obj *Generate) genFunctions() (map[string]*FunctionInfo, error) {
+	functions := make(map[string]*FunctionInfo)
+	if obj.DocsGenerateArgs.NoFunctions {
+		return functions, nil
+	}
+
+	m := funcs.Map() // map[string]func() interfaces.Func
+	names := []string{}
+	for name := range m {
+		names = append(names, name)
+	}
+	sort.Slice(names, func(i, j int) bool {
+		a := names[i]
+		b := names[j]
+		// TODO: do a sorted-by-package order.
+		return a < b
+	})
+
+	for _, name := range names {
+		//v := m[name]
+		//fn := v()
+		fn := m[name]()
+
+		// eg: golang/strings.has_suffix
+		sp := strings.Split(name, ".")
+		if len(sp) == 0 {
+			return nil, fmt.Errorf("unexpected empty function")
+		}
+		if len(sp) > 2 {
+			return nil, fmt.Errorf("unexpected function name: %s", name)
+		}
+		n := sp[0]
+		p := sp[0]
+		if len(sp) == 1 { // built-in
+			p = "" // no package!
+		}
+		if len(sp) == 2 { // normal import
+			n = sp[1]
+		}
+
+		if strings.HasPrefix(n, "_") {
+			// TODO: Should we display these somehow?
+			// built-in function
+			continue
+		}
+
+		var sig *string
+		//iface := ""
+		if x := fn.Info().Sig; x != nil {
+			s := x.String()
+			sig = &s
+			//iface = "simple"
+		}
+
+		metadata := &docsUtil.Metadata{}
+
+		// XXX: maybe we need a better way to get this?
+		mdFunc, ok := fn.(interfaces.MetadataFunc)
+		if !ok {
+			// Function doesn't tell us what the data is, let's try
+			// to get it automatically...
+			metadata.Typename = funcs.GetFunctionName(fn) // works!
+			metadata.Filename = ""                        // XXX: How can we get this?
+
+			// XXX: We only need this back-channel metadata store
+			// because we don't know how to get the filename without
+			// manually writing code in each function. Alternatively
+			// we could add a New() method to each struct and then
+			// we could modify the struct instead of having it be
+			// behind a copy which is needed to get new copies!
+			var err error
+			metadata, err = docsUtil.LookupFunction(name)
+			if err != nil {
+				return nil, err
+			}
+
+		} else if mdFunc == nil {
+			// programming error
+			return nil, fmt.Errorf("unexpected empty metadata for function: %s", name)
+
+		} else {
+			metadata = mdFunc.GetMetadata()
+		}
+
+		if metadata == nil {
+			return nil, fmt.Errorf("unexpected nil metadata for function: %s", name)
+		}
+
+		// This may be an empty func name if the function did not know
+		// how to get it. (This is normal for automatic regular funcs.)
+		if metadata.Typename == "" {
+			metadata.Typename = funcs.GetFunctionName(fn) // works!
+		}
+
+		fi, err := obj.getFunctionInfo(p, n, metadata)
+		if err != nil {
+			return nil, err
+		}
+		// We may not get any fields added if we can't find anything...
+		fi.Name = metadata.Typename
+		fi.Package = p
+		fi.Func = n
+		fi.File = metadata.Filename
+		//fi.Desc = desc
+		fi.Signature = sig
+
+		if fi.Func == "" {
+			return nil, fmt.Errorf("empty function name: %s", name)
+		}
+		if fi.File == "" {
+			return nil, fmt.Errorf("empty function file: %s", name)
+		}
+		if fi.Desc == "" {
+			obj.Logf("empty function desc: %s", name)
+		}
+		if fi.Signature == nil {
+			obj.Logf("empty function sig: %s", name)
+		}
+
+		functions[name] = fi
+	}
+
+	return functions, nil
+}
+
+// Output is the type of the final data that will be for the json output.
+type Output struct {
+	// Version is the sha1 or ref name of this specific version. This is
+	// used if we want to generate documentation with links matching the
+	// correct version. If unspecified then this assumes git master.
+	Version string `json:"version"`
+
+	// Resources contains the collection of every available resource!
+	// FIXME: should this be a list instead?
+	Resources map[string]*ResourceInfo `json:"resources"`
+
+	// Functions contains the collection of every available function!
+	// FIXME: should this be a list instead?
+	Functions map[string]*FunctionInfo `json:"functions"`
+}
+
+// ResourceInfo stores some information about each resource.
+type ResourceInfo struct {
+	// Name is the golang name of this resource.
+	Name string `json:"name"`
+
+	// Kind is the kind of this resource.
+	Kind string `json:"kind"`
+
+	// File is the file name where this resource exists.
+	File string `json:"file"`
+
+	// Desc explains what this resource does.
+	Desc string `json:"description"`
+
+	// Fields is a collection of each resource field and corresponding info.
+	Fields map[string]*ResourceFieldInfo `json:"fields"`
+}
+
+// ResourceFieldInfo stores some information about each field in each resource.
+type ResourceFieldInfo struct {
+	// Name is what this field is called in golang format.
+	Name string `json:"name"`
+
+	// Type is the mcl type for this field.
+	Type string `json:"type"`
+
+	// Desc explains what this field does.
+	Desc string `json:"description"`
+}
+
+// FunctionInfo stores some information about each function.
+type FunctionInfo struct {
+	// Name is the golang name of this function. This may be an actual
+	// function if used by the simple API, or the name of a struct.
+	Name string `json:"name"`
+
+	// Package is the import name to use to get to this function.
+	Package string `json:"package"`
+
+	// Func is the name of the function in that package.
+	Func string `json:"func"`
+
+	// File is the file name where this function exists.
+	File string `json:"file"`
+
+	// Desc explains what this function does.
+	Desc string `json:"description"`
+
+	// Signature is the type signature of this function. If empty then the
+	// signature is not known statically and it may be polymorphic.
+	Signature *string `json:"signature,omitempty"`
+}
+
+// commentCleaner takes a comment group and returns it as a clean string. It
+// removes the spurious newlines and programmer-focused comments. If there are
+// blank lines, it replaces them with a single newline. The idea is that the
+// webpage formatter would replace the newline with a <br /> or similar. This
+// code is a modified alternative of the ast.CommentGroup.Text() function.
+func commentCleaner(g *ast.CommentGroup) string {
+	if g == nil {
+		return ""
+	}
+	comments := make([]string, len(g.List))
+	for i, c := range g.List {
+		comments[i] = c.Text
+	}
+
+	lines := make([]string, 0, 10) // most comments are less than 10 lines
+	for _, c := range comments {
+		// Remove comment markers.
+		// The parser has given us exactly the comment text.
+		switch c[1] {
+		case '/':
+			//-style comment (no newline at the end)
+			c = c[2:]
+			if len(c) == 0 {
+				// empty line
+				break
+			}
+			if isDevComment(c[1:]) { // get rid of one space
+				continue
+			}
+			if c[0] == ' ' {
+				// strip first space - required for Example tests
+				c = c[1:]
+				break
+			}
+			//if isDirective(c) {
+			//	// Ignore //go:noinline, //line, and so on.
+			//	continue
+			//}
+		case '*':
+			/*-style comment */
+			c = c[2 : len(c)-2]
+		}
+
+		// Split on newlines.
+		cl := strings.Split(c, "\n")
+
+		// Walk lines, stripping trailing white space and adding to list.
+		for _, l := range cl {
+			lines = append(lines, stripTrailingWhitespace(l))
+		}
+	}
+
+	// Remove leading blank lines; convert runs of interior blank lines to a
+	// single blank line.
+	n := 0
+	for _, line := range lines {
+		if line != "" || n > 0 && lines[n-1] != "" {
+			lines[n] = line
+			n++
+		}
+	}
+	lines = lines[0:n]
+
+	// Concatenate all of these together. Blank lines should be a newline.
+	s := ""
+	for i, line := range lines {
+		if line == "" {
+			continue
+		}
+		s += line
+		if i < len(lines)-1 { // Is there another line?
+			if lines[i+1] == "" {
+				s += "\n" // Will eventually be a line break.
+			} else {
+				s += " "
+			}
+		}
+	}
+
+	return s
+}
+
+// TODO: should we use unicode.IsSpace instead?
+func isWhitespace(ch byte) bool { return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' }
+
+// TODO: should we replace with a strings package stdlib function?
+func stripTrailingWhitespace(s string) string {
+	i := len(s)
+	for i > 0 && isWhitespace(s[i-1]) {
+		i--
+	}
+	return s[0:i]
+}
+
+// isPrivate specifies if a field name is "private" or not.
+func isPrivate(fieldName string) bool {
+	if fieldName == "" {
+		panic("invalid field name")
+	}
+	x := fieldName[0:1]
+
+	if strings.ToLower(x) == x {
+		return true // it was already private
+	}
+
+	return false
+}
+
+// isDevComment tells us that the comment is for developers only!
+func isDevComment(comment string) bool {
+	if strings.HasPrefix(comment, "TODO:") {
+		return true
+	}
+	if strings.HasPrefix(comment, "FIXME:") {
+		return true
+	}
+	if strings.HasPrefix(comment, "XXX:") {
+		return true
+	}
+	return false
+}
+
+// safeVersion parses the main version string and returns a short hash for us.
+// For example, we might get a string of 0.0.26-176-gabcdef012-dirty as input,
+// and we'd want to return abcdef012.
+func safeVersion(version string) string {
+	const dirty = "-dirty"
+
+	s := version
+	if strings.HasSuffix(s, dirty) { // helpful dirty remover
+		s = s[0 : len(s)-len(dirty)]
+	}
+
+	ix := strings.LastIndex(s, "-")
+	if ix == -1 { // assume we have a standalone version (future proofing?)
+		return s
+	}
+	s = s[ix+1:]
+
+	// From the `git describe` man page: The "g" prefix stands for "git" and
+	// is used to allow describing the version of a software depending on
+	// the SCM the software is managed with. This is useful in an
+	// environment where people may use different SCMs.
+	const g = "g"
+	if strings.HasPrefix(s, g) {
+		s = s[len(g):]
+	}
+
+	return s
+}

docs/util/metadata.go

@@ -0,0 +1,103 @@
+// Mgmt
+// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+// Package util handles metadata for documentation generation.
+package util
+
+import (
+	"fmt"
+)
+
+var (
+	registeredResourceMetadata = make(map[string]*Metadata) // must initialize
+	registeredFunctionMetadata = make(map[string]*Metadata) // must initialize
+)
+
+// RegisterResource records the metadata for a resource of this kind.
+func RegisterResource(kind string, metadata *Metadata) error {
+	if _, exists := registeredResourceMetadata[kind]; exists {
+		return fmt.Errorf("metadata kind %s is already registered", kind)
+	}
+
+	registeredResourceMetadata[kind] = metadata
+
+	return nil
+}
+
+// LookupResource looks up the metadata for a resource of this kind.
+func LookupResource(kind string) (*Metadata, error) {
+	metadata, exists := registeredResourceMetadata[kind]
+	if !exists {
+		return nil, fmt.Errorf("not found")
+	}
+	return metadata, nil
+}
+
+// RegisterFunction records the metadata for a function of this name.
+func RegisterFunction(name string, metadata *Metadata) error {
+	if _, exists := registeredFunctionMetadata[name]; exists {
+		return fmt.Errorf("metadata named %s is already registered", name)
+	}
+
+	registeredFunctionMetadata[name] = metadata
+
+	return nil
+}
+
+// LookupFunction looks up the metadata for a function of this name.
+func LookupFunction(name string) (*Metadata, error) {
+	metadata, exists := registeredFunctionMetadata[name]
+	if !exists {
+		return nil, fmt.Errorf("not found")
+	}
+	return metadata, nil
+}
+
+// Metadata stores some additional information about the function or resource.
+// This is used to automatically generate documentation.
+type Metadata struct {
+	// Filename is the filename (without any base dir path) that this is in.
+	Filename string
+
+	// Typename is the string name of the main resource struct or function.
+	Typename string
+}
+
+// GetMetadata returns some metadata about the func. It can be called at any
+// time. This must not be named the same as the struct it's on or using it as an
+// anonymous embedded struct will stop us from being able to call this method.
+func (obj *Metadata) GetMetadata() *Metadata {
+	//if obj == nil { // TODO: Do I need this?
+	//	return nil
+	//}
+	return &Metadata{
+		Filename: obj.Filename,
+		Typename: obj.Typename,
+	}
+}

engine/resources.go

@@ -33,8 +33,12 @@ import (
 	"context"
 	"encoding/gob"
 	"fmt"
+	"path/filepath"
+	"reflect"
+	"runtime"
 	"strings"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/engine/local"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util/errwrap"
@@ -42,6 +46,12 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+const (
+	// ResourcesRelDir is the path where the resources are kept, relative to
+	// the main source code root.
+	ResourcesRelDir = "engine/resources/"
+)
+
 // TODO: should each resource be a sub-package?
 var registeredResources = map[string]func() Res{}
 
@@ -57,6 +67,23 @@ func RegisterResource(kind string, fn func() Res) {
 	}
 	gob.Register(f)
 	registeredResources[kind] = fn
+
+	// Additional metadata for documentation generation!
+	_, filename, _, ok := runtime.Caller(1)
+	if !ok {
+		panic(fmt.Sprintf("could not locate resource filename for %s", kind))
+	}
+	sp := strings.Split(reflect.TypeOf(f).String(), ".")
+	if len(sp) != 2 {
+		panic(fmt.Sprintf("could not parse resource struct name for %s", kind))
+	}
+
+	if err := docsUtil.RegisterResource(kind, &docsUtil.Metadata{
+		Filename: filepath.Base(filename),
+		Typename: sp[1],
+	}); err != nil {
+		panic(fmt.Sprintf("could not register resource metadata for %s", kind))
+	}
 }
 
 // RegisteredResourcesNames returns the kind of the registered resources.

engine/resources/msg.go

@@ -52,8 +52,14 @@ type MsgRes struct {
 
 	init *engine.Init
 
+	// Body is the body of the message to send.
 	Body string `lang:"body" yaml:"body"`
+
+	// Priority is the priority of the message. Currently this is one of:
+	// Emerg, Alert, Crit, Err, Warning, Notice, Info, Debug.
 	Priority string `lang:"priority" yaml:"priority"`
+
+	// Fields are the key/value pairs set in the journal if we are using it.
 	Fields map[string]string `lang:"fields" yaml:"fields"`
 
 	// Journal should be true to enable systemd journaled (journald) output.

engine/resources/noop.go

@@ -49,7 +49,8 @@ type NoopRes struct {
 
 	init *engine.Init
 
-	Comment string `lang:"comment" yaml:"comment"` // extra field for example purposes
+	// Comment is a useless comment field that you can use however you like.
+	Comment string `lang:"comment" yaml:"comment"`
 }
 
 // Default returns some sensible defaults for this resource.

engine/resources/print.go

@@ -52,7 +52,9 @@ type PrintRes struct {
 
 	init *engine.Init
 
-	Msg string `lang:"msg" yaml:"msg"` // the message to display
+	// Msg is the message to display.
+	Msg string `lang:"msg" yaml:"msg"`
+
 	// RefreshOnly is an option that causes the message to be printed only
 	// when notified by another resource. When set to true, this resource
 	// cannot be autogrouped.

engine/resources/test.go

@@ -70,6 +70,7 @@ type TestRes struct {
 	Uint64 uint64 `lang:"uint64" yaml:"uint64"`
 
 	//Uintptr uintptr `lang:"uintptr" yaml:"uintptr"`
+
 	Byte byte `lang:"byte" yaml:"byte"` // alias for uint8
 	Rune rune `lang:"rune" yaml:"rune"` // alias for int32, represents a Unicode code point
 
@@ -84,7 +85,7 @@ type TestRes struct {
 	Int8Ptr   *int8   `lang:"int8ptr" yaml:"int8ptr"`
 	Uint8Ptr  *uint8  `lang:"uint8ptr" yaml:"uint8ptr"`
 
-	// probably makes no sense, but is legal
+	// Int8PtrPtrPtr probably makes no sense, but is legal.
 	Int8PtrPtrPtr ***int8 `lang:"int8ptrptrptr" yaml:"int8ptrptrptr"`
 
 	SliceString []string          `lang:"slicestring" yaml:"slicestring"`

lang/core/datetime/print.go

@@ -42,7 +42,12 @@ func init() {
 	// FIXME: consider renaming this to printf, and add in a format string?
 	simple.ModuleRegister(ModuleName, "print", &simple.Scaffold{
 		T: types.NewType("func(a int) str"),
-		F: func(ctx context.Context, input []types.Value) (types.Value, error) {
+		F: Print,
+	})
+}
+
+// Print takes an epoch int and returns a string in unix format.
+func Print(ctx context.Context, input []types.Value) (types.Value, error) {
 	epochDelta := input[0].Int()
 	if epochDelta < 0 {
 		return nil, fmt.Errorf("epoch delta must be positive")
@@ -50,6 +55,4 @@ func init() {
 	return &types.StrValue{
 		V: time.Unix(epochDelta, 0).String(),
 	}, nil
-		},
-	})
 }

lang/core/example/answer.go

@@ -42,8 +42,12 @@ const Answer = 42
 func init() {
 	simple.ModuleRegister(ModuleName, "answer", &simple.Scaffold{
 		T: types.NewType("func() int"),
-		F: func(context.Context, []types.Value) (types.Value, error) {
-			return &types.IntValue{V: Answer}, nil
-		},
+		F: TheAnswerToLifeTheUniverseAndEverything,
 	})
 }
+
+// TheAnswerToLifeTheUniverseAndEverything returns the Answer to Life, the
+// Universe and Everything.
+func TheAnswerToLifeTheUniverseAndEverything(context.Context, []types.Value) (types.Value, error) {
+	return &types.IntValue{V: Answer}, nil
+}

lang/core/example/errorbool.go

@@ -40,13 +40,17 @@ import (
 func init() {
 	simple.ModuleRegister(ModuleName, "errorbool", &simple.Scaffold{
 		T: types.NewType("func(a bool) str"),
-		F: func(ctx context.Context, input []types.Value) (types.Value, error) {
+		F: ErrorBool,
+	})
+}
+
+// ErrorBool causes this function to error if you pass it true. Otherwise it
+// returns a string reminding you how to use it.
+func ErrorBool(ctx context.Context, input []types.Value) (types.Value, error) {
 	if input[0].Bool() {
 		return nil, fmt.Errorf("we errored on request")
 	}
 	return &types.StrValue{
 		V: "set input to true to generate an error",
 	}, nil
-		},
-	})
 }

lang/core/example/int2str.go

@@ -40,10 +40,13 @@ import (
 func init() {
 	simple.ModuleRegister(ModuleName, "int2str", &simple.Scaffold{
 		T: types.NewType("func(a int) str"),
-		F: func(ctx context.Context, input []types.Value) (types.Value, error) {
+		F: Int2Str,
+	})
+}
+
+// Int2Str takes an int, and returns it as a string.
+func Int2Str(ctx context.Context, input []types.Value) (types.Value, error) {
 	return &types.StrValue{
 		V: fmt.Sprintf("%d", input[0].Int()),
 	}, nil
-		},
-	})
 }

lang/core/example/str2int.go

@@ -40,7 +40,13 @@ import (
 func init() {
 	simple.ModuleRegister(ModuleName, "str2int", &simple.Scaffold{
 		T: types.NewType("func(a str) int"),
-		F: func(ctx context.Context, input []types.Value) (types.Value, error) {
+		F: Str2Int,
+	})
+}
+
+// Str2Int takes an str, and returns it as an int. If it can't convert it, it
+// returns 0.
+func Str2Int(ctx context.Context, input []types.Value) (types.Value, error) {
 	var i int64
 	if val, err := strconv.ParseInt(input[0].Str(), 10, 64); err == nil {
 		i = val
@@ -48,6 +54,4 @@ func init() {
 	return &types.IntValue{
 		V: i,
 	}, nil
-		},
-	})
 }

lang/core/math/fortytwo.go

@@ -65,6 +65,7 @@ func init() {
 		//	}
 		//	return nil, fmt.Errorf("can't use return type of: %s", typ.Out)
 		//},
+		D: FortyTwo, // get the docs from this
 	})
 }
 

lang/core/test/oneinstance_fact.go

@@ -132,6 +132,7 @@ func init() {
 			oneInstanceBMutex.Unlock()
 			return &types.StrValue{V: msg}, nil
 		},
+		D: &OneInstanceFact{},
 	})
 	simple.ModuleRegister(ModuleName, OneInstanceDFuncName, &simple.Scaffold{
 		T: types.NewType("func() str"),
@@ -144,6 +145,7 @@ func init() {
 			oneInstanceDMutex.Unlock()
 			return &types.StrValue{V: msg}, nil
 		},
+		D: &OneInstanceFact{},
 	})
 	simple.ModuleRegister(ModuleName, OneInstanceFFuncName, &simple.Scaffold{
 		T: types.NewType("func() str"),
@@ -156,6 +158,7 @@ func init() {
 			oneInstanceFMutex.Unlock()
 			return &types.StrValue{V: msg}, nil
 		},
+		D: &OneInstanceFact{},
 	})
 	simple.ModuleRegister(ModuleName, OneInstanceHFuncName, &simple.Scaffold{
 		T: types.NewType("func() str"),
@@ -168,6 +171,7 @@ func init() {
 			oneInstanceHMutex.Unlock()
 			return &types.StrValue{V: msg}, nil
 		},
+		D: &OneInstanceFact{},
 	})
 }
 

lang/funcs/facts/facts.go

@@ -51,10 +51,19 @@ func Register(name string, fn func() Fact) {
 	if _, ok := RegisteredFacts[name]; ok {
 		panic(fmt.Sprintf("a fact named %s is already registered", name))
 	}
+	f := fn()
+
+	metadata, err := funcs.GetFunctionMetadata(f)
+	if err != nil {
+		panic(fmt.Sprintf("could not locate fact filename for %s", name))
+	}
+
 	//gob.Register(fn())
 	funcs.Register(name, func() interfaces.Func { // implement in terms of func interface
 		return &FactFunc{
-			Fact: fn(),
+			Fact: f,
+
+			Metadata: metadata,
 		}
 	})
 	RegisteredFacts[name] = fn

lang/funcs/facts/func.go

@@ -33,6 +33,7 @@ import (
 	"context"
 	"fmt"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/lang/interfaces"
 	"github.com/purpleidea/mgmt/lang/types"
 )
@@ -40,6 +41,8 @@ import (
 // FactFunc is a wrapper for the fact interface. It implements the fact
 // interface in terms of Func to reduce the two down to a single mechanism.
 type FactFunc struct { // implements `interfaces.Func`
+	*docsUtil.Metadata
+
 	Fact Fact
 }
 

lang/funcs/funcs.go

@@ -33,9 +33,12 @@ package funcs
 import (
 	"context"
 	"fmt"
+	"reflect"
+	"runtime"
 	"strings"
 	"sync"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/lang/interfaces"
 	"github.com/purpleidea/mgmt/lang/types"
 	"github.com/purpleidea/mgmt/util/errwrap"
@@ -57,6 +60,10 @@ const (
 	// CoreDir is the directory prefix where core mcl code is embedded.
 	CoreDir = "core/"
 
+	// FunctionsRelDir is the path where the functions are kept, relative to
+	// the main source code root.
+	FunctionsRelDir = "lang/core/"
+
 	// ConcatFuncName is the name the concat function is registered as. It
 	// is listed here because it needs a well-known name that can be used by
 	// the string interpolation code.
@@ -119,6 +126,22 @@ func Register(name string, fn func() interfaces.Func) {
 
 	//gob.Register(fn())
 	registeredFuncs[name] = fn
+
+	f := fn() // Remember: If we modify this copy, it gets thrown away!
+
+	if _, ok := f.(interfaces.MetadataFunc); ok { // If it does it itself...
+		return
+	}
+
+	// We have to do it manually...
+	metadata, err := GetFunctionMetadata(f)
+	if err != nil {
+		panic(fmt.Sprintf("could not get function metadata for %s: %v", name, err))
+	}
+
+	if err := docsUtil.RegisterFunction(name, metadata); err != nil {
+		panic(fmt.Sprintf("could not register function metadata for %s", name))
+	}
 }
 
 // ModuleRegister is exactly like Register, except that it registers within a
@@ -178,6 +201,71 @@ func Map() map[string]func() interfaces.Func {
 	return m
 }
 
+// GetFunctionName reads the handle to find the underlying real function name.
+// The function can be an actual function or a struct which implements one.
+func GetFunctionName(fn interface{}) string {
+	pc := runtime.FuncForPC(reflect.ValueOf(fn).Pointer())
+	if pc == nil {
+		// This part works for structs, the other parts work for funcs.
+		t := reflect.TypeOf(fn)
+		if t.Kind() == reflect.Ptr {
+			t = t.Elem()
+		}
+
+		return t.Name()
+	}
+
+	// if pc.Name() is: github.com/purpleidea/mgmt/lang/core/math.Pow
+	sp := strings.Split(pc.Name(), "/")
+
+	// ...this will be: math.Pow
+	s := sp[len(sp)-1]
+
+	ix := strings.LastIndex(s, ".")
+	if ix == -1 { // standalone
+		return s
+	}
+
+	// ... this will be: Pow
+	return s[ix+1:]
+}
+
+// GetFunctionMetadata builds a metadata struct with everything about this func.
+func GetFunctionMetadata(fn interface{}) (*docsUtil.Metadata, error) {
+	nested := 1 // because this is wrapped in a function
+	// Additional metadata for documentation generation!
+	_, self, _, ok := runtime.Caller(0 + nested)
+	if !ok {
+		return nil, fmt.Errorf("could not locate function filename (1)")
+	}
+	depth := 1 + nested
+	// If this is ModuleRegister, we look deeper! Normal Register is depth 1
+	filename := self // initial condition to start the loop
+	for filename == self {
+		_, filename, _, ok = runtime.Caller(depth)
+		if !ok {
+			return nil, fmt.Errorf("could not locate function filename (2)")
+		}
+		depth++
+	}
+
+	// Get the function implementation path relative to FunctionsRelDir.
+	// FIXME: Technically we should split this by dirs instead of using
+	// string indexing, which is less correct, but we control the dirs.
+	ix := strings.LastIndex(filename, FunctionsRelDir)
+	if ix == -1 {
+		return nil, fmt.Errorf("could not locate function filename (3): %s", filename)
+	}
+	filename = filename[ix+len(FunctionsRelDir):]
+
+	funcname := GetFunctionName(fn)
+
+	return &docsUtil.Metadata{
+		Filename: filename,
+		Typename: funcname,
+	}, nil
+}
+
 // PureFuncExec is usually used to provisionally speculate about the result of a
 // pure function, by running it once, and returning the result. Pure functions
 // are expected to only produce one value that depends only on the input values.

lang/funcs/multi/multi.go

@@ -33,6 +33,7 @@ import (
 	"fmt"
 	"sort"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/lang/funcs"
 	"github.com/purpleidea/mgmt/lang/funcs/wrapped"
 	"github.com/purpleidea/mgmt/lang/interfaces"
@@ -60,6 +61,10 @@ type Scaffold struct {
 	// determined from the input types, then a different function API needs
 	// to be used. XXX: Should we extend this here?
 	M func(typ *types.Type) (interfaces.FuncSig, error)
+
+	// D is the documentation handle for this function. We look on that
+	// struct or function for the doc string.
+	D interface{}
 }
 
 // Register registers a simple, static, pure, polymorphic function. It is easier
@@ -92,9 +97,15 @@ func Register(name string, scaffold *Scaffold) {
 
 	RegisteredFuncs[name] = scaffold // store a copy for ourselves
 
+	metadata, err := funcs.GetFunctionMetadata(scaffold.D)
+	if err != nil {
+		panic(fmt.Sprintf("could not locate function filename for %s", name))
+	}
+
 	// register a copy in the main function database
 	funcs.Register(name, func() interfaces.Func {
 		return &Func{
+			Metadata: metadata,
 			WrappedFunc: &wrapped.Func{
 				Name: name,
 				// NOTE: It might be more correct to Copy here,
@@ -127,6 +138,7 @@ var _ interfaces.BuildableFunc = &Func{} // ensure it meets this expectation
 // function. This function API is unique in that it lets you provide your own
 // `Make` builder function to create the function implementation.
 type Func struct {
+	*docsUtil.Metadata
 	*WrappedFunc // *wrapped.Func as a type alias to pull in the base impl.
 
 	// Make is a build function to run after type unification. It will get

lang/funcs/operators/operators.go

@@ -36,6 +36,7 @@ import (
 	"fmt"
 	"math"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/lang/funcs"
 	"github.com/purpleidea/mgmt/lang/funcs/simple"
 	"github.com/purpleidea/mgmt/lang/interfaces"
@@ -468,6 +469,8 @@ func LookupOperator(operator string, size int) (*types.Type, error) {
 // OperatorFunc is an operator function that performs an operation on N values.
 // XXX: Can we wrap SimpleFunc instead of having the boilerplate here ourselves?
 type OperatorFunc struct {
+	*docsUtil.Metadata
+
 	Type *types.Type // Kind == Function, including operator arg
 
 	init *interfaces.Init

lang/funcs/simple/simple.go

@@ -35,6 +35,7 @@ import (
 	"reflect"
 	"strings"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/lang/funcs"
 	"github.com/purpleidea/mgmt/lang/funcs/wrapped"
 	"github.com/purpleidea/mgmt/lang/interfaces"
@@ -72,6 +73,11 @@ type Scaffold struct {
 	// can't be determined from the input types, then a different function
 	// API needs to be used. XXX: Should we extend this here?
 	F interfaces.FuncSig
+
+	// D is the documentation handle for this function. We look on that
+	// struct or function for the doc string instead of the F field if this
+	// is specified. (This is used for facts.)
+	D interface{}
 }
 
 // Register registers a simple, static, pure, polymorphic function. It is easier
@@ -105,9 +111,23 @@ func Register(name string, scaffold *Scaffold) {
 
 	RegisteredFuncs[name] = scaffold // store a copy for ourselves
 
+	// TODO: Do we need to special case either of these?
+	//if strings.HasPrefix(name, "embedded/") {}
+	//if strings.HasPrefix(name, "golang/") {}
+
+	var f interface{} = scaffold.F
+	if scaffold.D != nil { // override the doc lookup location if specified
+		f = scaffold.D
+	}
+	metadata, err := funcs.GetFunctionMetadata(f)
+	if err != nil {
+		panic(fmt.Sprintf("could not locate function filename for %s", name))
+	}
+
 	// register a copy in the main function database
 	funcs.Register(name, func() interfaces.Func {
 		return &Func{
+			Metadata: metadata,
 			WrappedFunc: &wrapped.Func{
 				Name: name,
 				// NOTE: It might be more correct to Copy here,
@@ -140,6 +160,7 @@ var _ interfaces.BuildableFunc = &Func{} // ensure it meets this expectation
 // function API, but that can run a very simple, static, pure, polymorphic
 // function.
 type Func struct {
+	*docsUtil.Metadata
 	*WrappedFunc // *wrapped.Func as a type alias to pull in the base impl.
 
 	// Check is a check function to run after type unification. It will get

lang/funcs/wrapped/wrapped.go

@@ -43,6 +43,8 @@ var _ interfaces.Func = &Func{} // ensure it meets this expectation
 // for the function API, but that can run a very simple, static, pure, function.
 // It can be wrapped by other structs that support polymorphism in various ways.
 type Func struct {
+	//*docsUtil.Metadata // This should NOT happen here, the parents do it.
+
 	// Name is a unique string name for the function.
 	Name string
 

lang/interfaces/func.go

@@ -34,6 +34,7 @@ import (
 	"fmt"
 	"strings"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/local"
 	"github.com/purpleidea/mgmt/lang/types"
@@ -279,6 +280,16 @@ type DataFunc interface {
 	SetData(*FuncData)
 }
 
+// MetadataFunc is a function that can return some extraneous information about
+// itself, which is usually used for documentation generation and so on.
+type MetadataFunc interface {
+	Func // implement everything in Func but add the additional requirements
+
+	// Metadata returns some metadata about the func. It can be called at
+	// any time, and doesn't require you run Init() or anything else first.
+	GetMetadata() *docsUtil.Metadata
+}
+
 // FuncEdge links an output vertex (value) to an input vertex with a named
 // argument.
 type FuncEdge struct {

test/test-docs-generate.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# check that our documentation still generates, even if we don't use it here
+
+# shellcheck disable=SC1091
+. test/util.sh
+
+echo running "$0"
+
+ROOT=$(dirname "${BASH_SOURCE}")/..
+cd "${ROOT}"
+
+failures=''
+
+run-test ./mgmt docs generate --output /tmp/docs.json &> /dev/null || fail_test "could not generate: $file"
+
+if [[ -n "$failures" ]]; then
+	echo 'FAIL'
+	echo "The following tests have failed:"
+	echo -e "$failures"
+	echo
+	exit 1
+fi
+echo 'PASS'