cli, docs: Add a docs command for doc generation

This took a lot longer than it looks to get right. It's not perfect, but
it now reliably generates documentation which we can put into gohugo.

lang/core/datetime/print.go

@@ -42,14 +42,17 @@ func init() {
 	// FIXME: consider renaming this to printf, and add in a format string?
 	simple.ModuleRegister(ModuleName, "print", &simple.Scaffold{
 		T: types.NewType("func(a int) str"),
-		F: func(ctx context.Context, input []types.Value) (types.Value, error) {
-			epochDelta := input[0].Int()
-			if epochDelta < 0 {
-				return nil, fmt.Errorf("epoch delta must be positive")
-			}
-			return &types.StrValue{
-				V: time.Unix(epochDelta, 0).String(),
-			}, nil
-		},
+		F: Print,
 	})
 }
+
+// Print takes an epoch int and returns a string in unix format.
+func Print(ctx context.Context, input []types.Value) (types.Value, error) {
+	epochDelta := input[0].Int()
+	if epochDelta < 0 {
+		return nil, fmt.Errorf("epoch delta must be positive")
+	}
+	return &types.StrValue{
+		V: time.Unix(epochDelta, 0).String(),
+	}, nil
+}

lang/core/example/answer.go

@@ -42,8 +42,12 @@ const Answer = 42
 func init() {
 	simple.ModuleRegister(ModuleName, "answer", &simple.Scaffold{
 		T: types.NewType("func() int"),
-		F: func(context.Context, []types.Value) (types.Value, error) {
-			return &types.IntValue{V: Answer}, nil
-		},
+		F: TheAnswerToLifeTheUniverseAndEverything,
 	})
 }
+
+// TheAnswerToLifeTheUniverseAndEverything returns the Answer to Life, the
+// Universe and Everything.
+func TheAnswerToLifeTheUniverseAndEverything(context.Context, []types.Value) (types.Value, error) {
+	return &types.IntValue{V: Answer}, nil
+}

lang/core/example/errorbool.go

@@ -40,13 +40,17 @@ import (
 func init() {
 	simple.ModuleRegister(ModuleName, "errorbool", &simple.Scaffold{
 		T: types.NewType("func(a bool) str"),
-		F: func(ctx context.Context, input []types.Value) (types.Value, error) {
-			if input[0].Bool() {
-				return nil, fmt.Errorf("we errored on request")
-			}
-			return &types.StrValue{
-				V: "set input to true to generate an error",
-			}, nil
-		},
+		F: ErrorBool,
 	})
 }
+
+// ErrorBool causes this function to error if you pass it true. Otherwise it
+// returns a string reminding you how to use it.
+func ErrorBool(ctx context.Context, input []types.Value) (types.Value, error) {
+	if input[0].Bool() {
+		return nil, fmt.Errorf("we errored on request")
+	}
+	return &types.StrValue{
+		V: "set input to true to generate an error",
+	}, nil
+}

lang/core/example/int2str.go

@@ -40,10 +40,13 @@ import (
 func init() {
 	simple.ModuleRegister(ModuleName, "int2str", &simple.Scaffold{
 		T: types.NewType("func(a int) str"),
-		F: func(ctx context.Context, input []types.Value) (types.Value, error) {
-			return &types.StrValue{
-				V: fmt.Sprintf("%d", input[0].Int()),
-			}, nil
-		},
+		F: Int2Str,
 	})
 }
+
+// Int2Str takes an int, and returns it as a string.
+func Int2Str(ctx context.Context, input []types.Value) (types.Value, error) {
+	return &types.StrValue{
+		V: fmt.Sprintf("%d", input[0].Int()),
+	}, nil
+}

lang/core/example/str2int.go

@@ -40,14 +40,18 @@ import (
 func init() {
 	simple.ModuleRegister(ModuleName, "str2int", &simple.Scaffold{
 		T: types.NewType("func(a str) int"),
-		F: func(ctx context.Context, input []types.Value) (types.Value, error) {
-			var i int64
-			if val, err := strconv.ParseInt(input[0].Str(), 10, 64); err == nil {
-				i = val
-			}
-			return &types.IntValue{
-				V: i,
-			}, nil
-		},
+		F: Str2Int,
 	})
 }
+
+// Str2Int takes an str, and returns it as an int. If it can't convert it, it
+// returns 0.
+func Str2Int(ctx context.Context, input []types.Value) (types.Value, error) {
+	var i int64
+	if val, err := strconv.ParseInt(input[0].Str(), 10, 64); err == nil {
+		i = val
+	}
+	return &types.IntValue{
+		V: i,
+	}, nil
+}

lang/core/math/fortytwo.go

@@ -65,6 +65,7 @@ func init() {
 		//	}
 		//	return nil, fmt.Errorf("can't use return type of: %s", typ.Out)
 		//},
+		D: FortyTwo, // get the docs from this
 	})
 }
 

lang/core/test/oneinstance_fact.go

@@ -132,6 +132,7 @@ func init() {
 			oneInstanceBMutex.Unlock()
 			return &types.StrValue{V: msg}, nil
 		},
+		D: &OneInstanceFact{},
 	})
 	simple.ModuleRegister(ModuleName, OneInstanceDFuncName, &simple.Scaffold{
 		T: types.NewType("func() str"),
@@ -144,6 +145,7 @@ func init() {
 			oneInstanceDMutex.Unlock()
 			return &types.StrValue{V: msg}, nil
 		},
+		D: &OneInstanceFact{},
 	})
 	simple.ModuleRegister(ModuleName, OneInstanceFFuncName, &simple.Scaffold{
 		T: types.NewType("func() str"),
@@ -156,6 +158,7 @@ func init() {
 			oneInstanceFMutex.Unlock()
 			return &types.StrValue{V: msg}, nil
 		},
+		D: &OneInstanceFact{},
 	})
 	simple.ModuleRegister(ModuleName, OneInstanceHFuncName, &simple.Scaffold{
 		T: types.NewType("func() str"),
@@ -168,6 +171,7 @@ func init() {
 			oneInstanceHMutex.Unlock()
 			return &types.StrValue{V: msg}, nil
 		},
+		D: &OneInstanceFact{},
 	})
 }
 

lang/funcs/facts/facts.go

@@ -51,10 +51,19 @@ func Register(name string, fn func() Fact) {
 	if _, ok := RegisteredFacts[name]; ok {
 		panic(fmt.Sprintf("a fact named %s is already registered", name))
 	}
+	f := fn()
+
+	metadata, err := funcs.GetFunctionMetadata(f)
+	if err != nil {
+		panic(fmt.Sprintf("could not locate fact filename for %s", name))
+	}
+
 	//gob.Register(fn())
 	funcs.Register(name, func() interfaces.Func { // implement in terms of func interface
 		return &FactFunc{
-			Fact: fn(),
+			Fact: f,
+
+			Metadata: metadata,
 		}
 	})
 	RegisteredFacts[name] = fn

lang/funcs/facts/func.go

@@ -33,6 +33,7 @@ import (
 	"context"
 	"fmt"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/lang/interfaces"
 	"github.com/purpleidea/mgmt/lang/types"
 )
@@ -40,6 +41,8 @@ import (
 // FactFunc is a wrapper for the fact interface. It implements the fact
 // interface in terms of Func to reduce the two down to a single mechanism.
 type FactFunc struct { // implements `interfaces.Func`
+	*docsUtil.Metadata
+
 	Fact Fact
 }
 

lang/funcs/funcs.go

@@ -33,9 +33,12 @@ package funcs
 import (
 	"context"
 	"fmt"
+	"reflect"
+	"runtime"
 	"strings"
 	"sync"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/lang/interfaces"
 	"github.com/purpleidea/mgmt/lang/types"
 	"github.com/purpleidea/mgmt/util/errwrap"
@@ -57,6 +60,10 @@ const (
 	// CoreDir is the directory prefix where core mcl code is embedded.
 	CoreDir = "core/"
 
+	// FunctionsRelDir is the path where the functions are kept, relative to
+	// the main source code root.
+	FunctionsRelDir = "lang/core/"
+
 	// ConcatFuncName is the name the concat function is registered as. It
 	// is listed here because it needs a well-known name that can be used by
 	// the string interpolation code.
@@ -119,6 +126,22 @@ func Register(name string, fn func() interfaces.Func) {
 
 	//gob.Register(fn())
 	registeredFuncs[name] = fn
+
+	f := fn() // Remember: If we modify this copy, it gets thrown away!
+
+	if _, ok := f.(interfaces.MetadataFunc); ok { // If it does it itself...
+		return
+	}
+
+	// We have to do it manually...
+	metadata, err := GetFunctionMetadata(f)
+	if err != nil {
+		panic(fmt.Sprintf("could not get function metadata for %s: %v", name, err))
+	}
+
+	if err := docsUtil.RegisterFunction(name, metadata); err != nil {
+		panic(fmt.Sprintf("could not register function metadata for %s", name))
+	}
 }
 
 // ModuleRegister is exactly like Register, except that it registers within a
@@ -178,6 +201,71 @@ func Map() map[string]func() interfaces.Func {
 	return m
 }
 
+// GetFunctionName reads the handle to find the underlying real function name.
+// The function can be an actual function or a struct which implements one.
+func GetFunctionName(fn interface{}) string {
+	pc := runtime.FuncForPC(reflect.ValueOf(fn).Pointer())
+	if pc == nil {
+		// This part works for structs, the other parts work for funcs.
+		t := reflect.TypeOf(fn)
+		if t.Kind() == reflect.Ptr {
+			t = t.Elem()
+		}
+
+		return t.Name()
+	}
+
+	// if pc.Name() is: github.com/purpleidea/mgmt/lang/core/math.Pow
+	sp := strings.Split(pc.Name(), "/")
+
+	// ...this will be: math.Pow
+	s := sp[len(sp)-1]
+
+	ix := strings.LastIndex(s, ".")
+	if ix == -1 { // standalone
+		return s
+	}
+
+	// ... this will be: Pow
+	return s[ix+1:]
+}
+
+// GetFunctionMetadata builds a metadata struct with everything about this func.
+func GetFunctionMetadata(fn interface{}) (*docsUtil.Metadata, error) {
+	nested := 1 // because this is wrapped in a function
+	// Additional metadata for documentation generation!
+	_, self, _, ok := runtime.Caller(0 + nested)
+	if !ok {
+		return nil, fmt.Errorf("could not locate function filename (1)")
+	}
+	depth := 1 + nested
+	// If this is ModuleRegister, we look deeper! Normal Register is depth 1
+	filename := self // initial condition to start the loop
+	for filename == self {
+		_, filename, _, ok = runtime.Caller(depth)
+		if !ok {
+			return nil, fmt.Errorf("could not locate function filename (2)")
+		}
+		depth++
+	}
+
+	// Get the function implementation path relative to FunctionsRelDir.
+	// FIXME: Technically we should split this by dirs instead of using
+	// string indexing, which is less correct, but we control the dirs.
+	ix := strings.LastIndex(filename, FunctionsRelDir)
+	if ix == -1 {
+		return nil, fmt.Errorf("could not locate function filename (3): %s", filename)
+	}
+	filename = filename[ix+len(FunctionsRelDir):]
+
+	funcname := GetFunctionName(fn)
+
+	return &docsUtil.Metadata{
+		Filename: filename,
+		Typename: funcname,
+	}, nil
+}
+
 // PureFuncExec is usually used to provisionally speculate about the result of a
 // pure function, by running it once, and returning the result. Pure functions
 // are expected to only produce one value that depends only on the input values.

lang/funcs/multi/multi.go

@@ -33,6 +33,7 @@ import (
 	"fmt"
 	"sort"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/lang/funcs"
 	"github.com/purpleidea/mgmt/lang/funcs/wrapped"
 	"github.com/purpleidea/mgmt/lang/interfaces"
@@ -60,6 +61,10 @@ type Scaffold struct {
 	// determined from the input types, then a different function API needs
 	// to be used. XXX: Should we extend this here?
 	M func(typ *types.Type) (interfaces.FuncSig, error)
+
+	// D is the documentation handle for this function. We look on that
+	// struct or function for the doc string.
+	D interface{}
 }
 
 // Register registers a simple, static, pure, polymorphic function. It is easier
@@ -92,9 +97,15 @@ func Register(name string, scaffold *Scaffold) {
 
 	RegisteredFuncs[name] = scaffold // store a copy for ourselves
 
+	metadata, err := funcs.GetFunctionMetadata(scaffold.D)
+	if err != nil {
+		panic(fmt.Sprintf("could not locate function filename for %s", name))
+	}
+
 	// register a copy in the main function database
 	funcs.Register(name, func() interfaces.Func {
 		return &Func{
+			Metadata: metadata,
 			WrappedFunc: &wrapped.Func{
 				Name: name,
 				// NOTE: It might be more correct to Copy here,
@@ -127,6 +138,7 @@ var _ interfaces.BuildableFunc = &Func{} // ensure it meets this expectation
 // function. This function API is unique in that it lets you provide your own
 // `Make` builder function to create the function implementation.
 type Func struct {
+	*docsUtil.Metadata
 	*WrappedFunc // *wrapped.Func as a type alias to pull in the base impl.
 
 	// Make is a build function to run after type unification. It will get

lang/funcs/operators/operators.go

@@ -36,6 +36,7 @@ import (
 	"fmt"
 	"math"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/lang/funcs"
 	"github.com/purpleidea/mgmt/lang/funcs/simple"
 	"github.com/purpleidea/mgmt/lang/interfaces"
@@ -468,6 +469,8 @@ func LookupOperator(operator string, size int) (*types.Type, error) {
 // OperatorFunc is an operator function that performs an operation on N values.
 // XXX: Can we wrap SimpleFunc instead of having the boilerplate here ourselves?
 type OperatorFunc struct {
+	*docsUtil.Metadata
+
 	Type *types.Type // Kind == Function, including operator arg
 
 	init *interfaces.Init

lang/funcs/simple/simple.go

@@ -35,6 +35,7 @@ import (
 	"reflect"
 	"strings"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/lang/funcs"
 	"github.com/purpleidea/mgmt/lang/funcs/wrapped"
 	"github.com/purpleidea/mgmt/lang/interfaces"
@@ -72,6 +73,11 @@ type Scaffold struct {
 	// can't be determined from the input types, then a different function
 	// API needs to be used. XXX: Should we extend this here?
 	F interfaces.FuncSig
+
+	// D is the documentation handle for this function. We look on that
+	// struct or function for the doc string instead of the F field if this
+	// is specified. (This is used for facts.)
+	D interface{}
 }
 
 // Register registers a simple, static, pure, polymorphic function. It is easier
@@ -105,9 +111,23 @@ func Register(name string, scaffold *Scaffold) {
 
 	RegisteredFuncs[name] = scaffold // store a copy for ourselves
 
+	// TODO: Do we need to special case either of these?
+	//if strings.HasPrefix(name, "embedded/") {}
+	//if strings.HasPrefix(name, "golang/") {}
+
+	var f interface{} = scaffold.F
+	if scaffold.D != nil { // override the doc lookup location if specified
+		f = scaffold.D
+	}
+	metadata, err := funcs.GetFunctionMetadata(f)
+	if err != nil {
+		panic(fmt.Sprintf("could not locate function filename for %s", name))
+	}
+
 	// register a copy in the main function database
 	funcs.Register(name, func() interfaces.Func {
 		return &Func{
+			Metadata: metadata,
 			WrappedFunc: &wrapped.Func{
 				Name: name,
 				// NOTE: It might be more correct to Copy here,
@@ -140,6 +160,7 @@ var _ interfaces.BuildableFunc = &Func{} // ensure it meets this expectation
 // function API, but that can run a very simple, static, pure, polymorphic
 // function.
 type Func struct {
+	*docsUtil.Metadata
 	*WrappedFunc // *wrapped.Func as a type alias to pull in the base impl.
 
 	// Check is a check function to run after type unification. It will get

lang/funcs/wrapped/wrapped.go

@@ -43,6 +43,8 @@ var _ interfaces.Func = &Func{} // ensure it meets this expectation
 // for the function API, but that can run a very simple, static, pure, function.
 // It can be wrapped by other structs that support polymorphism in various ways.
 type Func struct {
+	//*docsUtil.Metadata // This should NOT happen here, the parents do it.
+
 	// Name is a unique string name for the function.
 	Name string
 

lang/interfaces/func.go

@@ -34,6 +34,7 @@ import (
 	"fmt"
 	"strings"
 
+	docsUtil "github.com/purpleidea/mgmt/docs/util"
 	"github.com/purpleidea/mgmt/engine"
 	"github.com/purpleidea/mgmt/engine/local"
 	"github.com/purpleidea/mgmt/lang/types"
@@ -279,6 +280,16 @@ type DataFunc interface {
 	SetData(*FuncData)
 }
 
+// MetadataFunc is a function that can return some extraneous information about
+// itself, which is usually used for documentation generation and so on.
+type MetadataFunc interface {
+	Func // implement everything in Func but add the additional requirements
+
+	// Metadata returns some metadata about the func. It can be called at
+	// any time, and doesn't require you run Init() or anything else first.
+	GetMetadata() *docsUtil.Metadata
+}
+
 // FuncEdge links an output vertex (value) to an input vertex with a named
 // argument.
 type FuncEdge struct {