lang: funcs: Autogenerated a lot of new functions

Signed-off-by: Julien Pivotto <roidelapluie@inuits.eu>

Makefile

@@ -504,14 +504,10 @@ help: ## show this help screen
 		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 	@echo ''
 
-funcgen: lang/funcs/core/generated_funcs_test.go lang/funcs/core/generated_funcs.go
-
-lang/funcs/core/generated_funcs_test.go: lang/funcs/funcgen/*.go lang/funcs/core/funcgen.yaml lang/funcs/funcgen/templates/generated_funcs_test.go.tpl
-	@echo "Generating: funcs test..."
-	@go run lang/funcs/funcgen/*.go -templates lang/funcs/funcgen/templates/generated_funcs_test.go.tpl 2>/dev/null
+funcgen: lang/funcs/core/generated_funcs.go
 
 lang/funcs/core/generated_funcs.go: lang/funcs/funcgen/*.go lang/funcs/core/funcgen.yaml lang/funcs/funcgen/templates/generated_funcs.go.tpl
 	@echo "Generating: funcs..."
-	@go run lang/funcs/funcgen/*.go -templates lang/funcs/funcgen/templates/generated_funcs.go.tpl 2>/dev/null
+	@go run `find lang/funcs/funcgen/ -maxdepth 1 -type f -name '*.go' -not -name '*_test.go'` -templates=lang/funcs/funcgen/templates/generated_funcs.go.tpl 2>/dev/null
 
 # vim: ts=8

examples/lang/html.mcl

@@ -0,0 +1,10 @@
+import "golang/html"
+import "fmt"
+
+$text1 = html.unescape_string("<h1>MGMT!</h1>")
+$text2 = html.escape_string("Test & Re-Test\n")
+
+file "/tmp/index.html" {
+	state => "exists",
+	content => "${text1}${text2}",
+}

examples/lang/os.mcl

@@ -0,0 +1,27 @@
+import "golang/os"
+import "golang/exec"
+import "fmt"
+
+$tmpdir = os.temp_dir()
+
+file "${tmpdir}/execinfo" {
+	state => "exists",
+	content => fmt.printf("mgmt is at %s\n", os.executable()),
+}
+
+file "${tmpdir}/mgmtenv" {
+	state => "exists",
+	content => os.expand_env("$HOME sweet ${os.getenv(\"HOME\")}\n"),
+}
+
+file "${tmpdir}/mgmtos" {
+	state => "exists",
+	content => os.readlink("/bin"),
+}
+
+$rm = exec.look_path("rm")
+
+file "${tmpdir}/cache" {
+	state => "exists",
+	content => "Plz cache in ${os.user_cache_dir()}.\nYour home is ${os.user_home_dir()}. Remove with ${rm}\n",
+}

examples/lang/runtime.mcl

@@ -0,0 +1,7 @@
+import "golang/runtime"
+import "fmt"
+
+file "/tmp/mgmtinfo" {
+	state => "exists",
+	content => fmt.printf("Hi from mgmt! mgmt is running with %s; and GOROOT is %s.\n", runtime.version(), runtime.goroot()),
+}

lang/funcs/core/funcgen.yaml

@@ -1,55 +1,19 @@
 # This file is used by github.com/purpleidea/mgmt/lang/funcs/funcgen/ to
 # generate mcl functions.
-functions:
-- mgmtName: to_upper
-  mgmtPackage: strings
-  help: turns a string to uppercase.
-  goPackage: strings
-  goFunc: ToUpper
-  args: [{name: a, type: string}]
-  return: [{type: string}]
-  tests:
-    - args: [{type: string, value: "Hello"}]
-      return: [{type: string, value: "HELLO"}]
-    - args: [{type: string, value: "HELLO 22"}]
-      return: [{type: string, value: "HELLO 22"}]
-- mgmtName: trim
-  mgmtPackage: strings
-  help: returns a slice of the string s with all leading and trailing Unicode code points contained in cutset removed.
-  goPackage: strings
-  goFunc: Trim
-  args: [{name: s, type: string}, {name: cutset, type: string}]
-  return: [{type: string}]
-  tests:
-    - args: [{type: string, value: "??Hello.."}, {type: string, value: "?."}]
-      return: [{type: string, value: "Hello"}]
-- mgmtName: trim_left
-  mgmtPackage: strings
-  help: returns a slice of the string s with all leading Unicode code points contained in cutset removed.
-  goPackage: strings
-  goFunc: TrimLeft
-  args: [{name: s, type: string}, {name: cutset, type: string}]
-  return: [{type: string}]
-  tests:
-    - args: [{type: string, value: "??Hello.."}, {type: string, value: "?."}]
-      return: [{type: string, value: "Hello.."}]
-- mgmtName: trim_space
-  mgmtPackage: strings
-  help: returns a slice of the string s, with all leading and trailing white space removed, as defined by Unicode.
-  goPackage: strings
-  goFunc: TrimSpace
-  args: [{name: s, type: string}]
-  return: [{type: string}]
-  tests:
-    - args: [{type: string, value: "Hello 2    "}]
-      return: [{type: string, value: "Hello 2"}]
-- mgmtName: trim_right
-  mgmtPackage: strings
-  help: returns a slice of the string s with all trailing Unicode code points contained in cutset removed.
-  goPackage: strings
-  goFunc: TrimRight
-  args: [{name: s, type: string}, {name: cutset, type: string}]
-  return: [{type: string}]
-  tests:
-    - args: [{type: string, value: "??Hello.."}, {type: string, value: "?."}]
-      return: [{type: string, value: "??Hello"}]
+packages:
+- name: html
+- name: math
+- name: math/rand
+  alias: rand
+  mgmtAlias: rand
+- name: os
+- name: os/exec
+  alias: exec
+  mgmtAlias: exec
+- name: path
+- name: path/filepath
+  alias: filepath
+  mgmtAlias: filepath
+- name: runtime
+- name: strconv
+- name: strings

lang/funcs/funcgen/config.go

@@ -24,45 +24,78 @@ import (
 )
 
 type config struct {
-	Functions functions `yaml:"functions"`
+	Packages golangPackages `yaml:"packages"`
 }
 
 type functions []function
 
-type testarg struct {
+type arg struct {
+	// Name is the name of the argument.
 	Name string `yaml:"name,omitempty"`
+	// Value is the value of the argument.
+	Value string `yaml:"value,omitempty"`
+	// Type is the type of the argument.
+	// Supported: bool, string, int, int64, float64.
 	Type string `yaml:"type"`
-	Value string `yaml:"value"`
 }
 
-type arg struct {
-	Name string `yaml:"name,omitempty"`
-	Type string `yaml:"type"`
+// GolangType prints the golang equivalent of a mcl type.
+func (obj *arg) GolangType() string {
+	t := obj.Type
+	if t == "float" {
+		return "float64"
+	}
+	return t
 }
 
 // ToMcl prints the arg signature as expected by mcl.
 func (obj *arg) ToMcl() (string, error) {
-	if obj.Type == "string" {
+	var prefix string
 	if obj.Name != "" {
-			return fmt.Sprintf("%s str", obj.Name), nil
-		}
-		return types.TypeStr.String(), nil
+		prefix = fmt.Sprintf("%s ", obj.Name)
 	}
+	switch obj.Type {
+	case "bool":
+		return fmt.Sprintf("%s%s", prefix, types.TypeBool.String()), nil
+	case "string":
+		return fmt.Sprintf("%s%s", prefix, types.TypeStr.String()), nil
+	case "int64", "int":
+		return fmt.Sprintf("%s%s", prefix, types.TypeInt.String()), nil
+	case "float64":
+		return fmt.Sprintf("%s%s", prefix, types.TypeFloat.String()), nil
+	default:
 		return "", fmt.Errorf("cannot convert %v to mcl", obj)
 	}
+}
 
 // ToGo prints the arg signature as expected by golang.
-func (obj *arg) ToGo() (string, error) {
-	if obj.Type == "string" {
+func (obj *arg) ToGolang() (string, error) {
+	switch obj.Type {
+	case "bool":
+		return "Bool", nil
+	case "string":
 		return "Str", nil
+	case "int", "int64":
+		return "Int", nil
+	case "float64":
+		return "Float", nil
+	default:
+		return "", fmt.Errorf("cannot convert %v to golang", obj)
 	}
-	return "", fmt.Errorf("cannot convert %v to go", obj)
 }
 
 // ToTestInput prints the arg signature as expected by tests.
 func (obj *arg) ToTestInput() (string, error) {
-	if obj.Type == "string" {
+	switch obj.Type {
+	case "bool":
+		return fmt.Sprintf("&types.BoolValue{V: %s}", obj.Name), nil
+	case "string":
 		return fmt.Sprintf("&types.StrValue{V: %s}", obj.Name), nil
-	}
+	case "int":
+		return fmt.Sprintf("&types.IntValue{V: %s}", obj.Name), nil
+	case "float":
+		return fmt.Sprintf("&types.FloatValue{V: %s}", obj.Name), nil
+	default:
 		return "", fmt.Errorf("cannot convert %v to test input", obj)
 	}
+}

lang/funcs/funcgen/fixtures/.gitignore

@@ -0,0 +1 @@
+*.result

lang/funcs/funcgen/fixtures/func_base.tpl

@@ -0,0 +1,83 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"testpkg"
+
+	"github.com/purpleidea/mgmt/lang/funcs/simple"
+	"github.com/purpleidea/mgmt/lang/types"
+)
+
+func init() {
+	simple.ModuleRegister("golang/testpkg", "all_kind", &types.FuncValue{
+		T: types.NewType("func(x int, y str) float"),
+		V: TestpkgAllKind,
+	})
+	simple.ModuleRegister("golang/testpkg", "to_upper", &types.FuncValue{
+		T: types.NewType("func(s str) str"),
+		V: TestpkgToUpper,
+	})
+	simple.ModuleRegister("golang/testpkg", "max", &types.FuncValue{
+		T: types.NewType("func(x float, y float) float"),
+		V: TestpkgMax,
+	})
+	simple.ModuleRegister("golang/testpkg", "with_error", &types.FuncValue{
+		T: types.NewType("func(s str) str"),
+		V: TestpkgWithError,
+	})
+	simple.ModuleRegister("golang/testpkg", "with_int", &types.FuncValue{
+		T: types.NewType("func(s float, i int, x int, j int, k int, b bool, t str) str"),
+		V: TestpkgWithInt,
+	})
+
+}
+
+func TestpkgAllKind(input []types.Value) (types.Value, error) {
+	return &types.FloatValue{
+		V: testpkg.AllKind(input[0].Int(), input[1].Str()),
+	}, nil
+}
+
+func TestpkgToUpper(input []types.Value) (types.Value, error) {
+	return &types.StrValue{
+		V: testpkg.ToUpper(input[0].Str()),
+	}, nil
+}
+
+func TestpkgMax(input []types.Value) (types.Value, error) {
+	return &types.FloatValue{
+		V: testpkg.Max(input[0].Float(), input[1].Float()),
+	}, nil
+}
+
+func TestpkgWithError(input []types.Value) (types.Value, error) {
+	v, err := testpkg.WithError(input[0].Str())
+	if err != nil {
+		return nil, err
+	}
+	return &types.StrValue{
+		V: v,
+	}, nil
+}
+
+func TestpkgWithInt(input []types.Value) (types.Value, error) {
+	return &types.StrValue{
+		V: testpkg.WithInt(input[0].Float(), int(input[1].Int()), input[2].Int(), int(input[3].Int()), int(input[4].Int()), input[5].Bool(), input[6].Str()),
+	}, nil
+}

lang/funcs/funcgen/fixtures/func_base.txt

@@ -0,0 +1,14 @@
+Random
+const E = 2.71828182845904523536028747135266249775724709369995957496696763 ...
+const MaxFloat32 = 3.40282346638528859811704183484516925440e+38 ...
+const MaxInt8 = 1<<7 - 1 ...
+func Lgamma(x float64) (lgamma float64, sign int)
+func AllKind(x int64, y string) float64
+func ToUpper(s string) string
+func ToLower(s string) string
+func Max(x, y float64) float64
+func WithError(s string) (string, error)
+func WithErrorButNothingElse(s string) error
+func WithNothingElse(s string)
+func Nextafter32(x, y float32) (r float32)
+func WithInt(s float64, i int, x int64, j, k int, b bool, t string) string

lang/funcs/funcgen/fixtures/func_base.yaml

@@ -0,0 +1,42 @@
+- mgmtPackage: golang/testpkg
+  mgmtName: all_kind
+  internalName: TestpkgAllKind
+  golangPackage: &pkg
+    name: testpkg
+    exclude: [ToLower]
+  golangFunc: AllKind
+  errorful: false
+  args: [{name: x, type: int64},{name: y, type: string}]
+  return: [{type: float64}]
+- mgmtPackage: golang/testpkg
+  mgmtName: to_upper
+  internalName: TestpkgToUpper
+  golangPackage: *pkg
+  golangFunc: ToUpper
+  errorful: false
+  args: [{name: s, type: string}]
+  return: [{type: string}]
+- mgmtPackage: golang/testpkg
+  mgmtName: max
+  internalName: TestpkgMax
+  golangPackage: *pkg
+  golangFunc: Max
+  errorful: false
+  args: [{name: x, type: float64},{name: y, type: float64}]
+  return: [{type: float64}]
+- mgmtPackage: golang/testpkg
+  mgmtName: with_error
+  internalName: TestpkgWithError
+  golangPackage: *pkg
+  golangFunc: WithError
+  errorful: true
+  args: [{name: s, type: string}]
+  return: [{type: string}]
+- mgmtPackage: golang/testpkg
+  mgmtName: with_int
+  internalName: TestpkgWithInt
+  golangPackage: *pkg
+  golangFunc: WithInt
+  errorful: false
+  args: [{name: s, type: float64}, {name: i, type: int}, {name: x, type: int64}, {name: j, type: int}, {name: k, type: int}, {name: b, type: bool}, {name: t, type: string}]
+  return: [{type: string}]

lang/funcs/funcgen/func.go

@@ -28,34 +28,40 @@ import (
 )
 
 type function struct {
+	// MclName is the name of the package of the function in mcl.
 	MgmtPackage string `yaml:"mgmtPackage"`
-	MgmtName    string     `yaml:"mgmtName"`
+	// MclName is the name of the function in mcl.
+	MclName string `yaml:"mgmtName"`
+	// InternalName is the name used inside the templated file.
+	// Used to avoid clash between same functions from different packages.
+	InternalName string `yaml:"internalName"`
+	// Help is the docstring of the function, including // and
+	// new lines.
 	Help string `yaml:"help"`
-	GoPackage   string     `yaml:"goPackage"`
-	GoFunc      string     `yaml:"goFunc"`
+	// GolangPackage is the representation of the package.
+	GolangPackage *golangPackage `yaml:"golangPackage"`
+	// GolangFunc is the name of the function in golang.
+	GolangFunc string `yaml:"golangFunc"`
+	// Errorful indicates wether the golang function can return an error
+	// as second argument.
+	Errorful bool `yaml:"errorful"`
+	// Args is the list of the arguments of the function.
 	Args []arg `yaml:"args"`
+	// ExtraGolangArgs are arguments that are added at the end of the go call.
+	// e.g. strconv.ParseFloat("3.1415", 64) could require add 64.
+	ExtraGolangArgs []arg `yaml:"extraGolangArgs"`
+	// Return is the list of arguments returned by the function.
 	Return []arg `yaml:"return"`
-	Tests       []functest `yaml:"tests"`
 }
 
-type functest struct {
-	Args   []testarg `yaml:"args"`
-	Expect []testarg `yaml:"return"`
-}
-
-type templateInput struct {
-	Func        function
-	MgmtPackage string
-}
-
-func parseFuncs(c config, path, templates string) error {
+func parseFuncs(c config, f functions, path, templates string) error {
 	templateFiles, err := filepath.Glob(templates)
 	if err != nil {
 		return err
 	}
 	for _, tpl := range templateFiles {
 		log.Printf("Template: %s", tpl)
-		err = generateTemplate(c, path, tpl)
+		err = generateTemplate(c, f, path, tpl, "")
 		if err != nil {
 			return err
 		}
@@ -63,7 +69,7 @@ func parseFuncs(c config, path, templates string) error {
 	return nil
 }
 
-func generateTemplate(c config, path, templateFile string) error {
+func generateTemplate(c config, f functions, path, templateFile, finalName string) error {
 	log.Printf("Reading: %s", templateFile)
 	basename := filepath.Base(templateFile)
 	tplFile, err := ioutil.ReadFile(templateFile)
@@ -74,28 +80,42 @@ func generateTemplate(c config, path, templateFile string) error {
 	if err != nil {
 		return err
 	}
-	finalName := strings.TrimSuffix(basename, ".tpl")
+	if finalName == "" {
+		finalName = strings.TrimSuffix(basename, ".tpl")
+	}
 	finalPath := filepath.Join(path, finalName)
-	log.Printf("Writing: %s", finalPath)
 	finalFile, err := os.Create(finalPath)
 	if err != nil {
 		return err
 	}
-	if err = t.Execute(finalFile, c); err != nil {
+	if err = t.Execute(finalFile, struct {
+		Packages  golangPackages
+		Functions []function
+	}{
+		c.Packages,
+		f,
+	}); err != nil {
 		return err
 	}
 	return nil
 }
 
-// MakeGoArgs translates the func args to go args.
-func (obj *function) MakeGoArgs() (string, error) {
+// MakeGolangArgs translates the func args to golang args.
+func (obj *function) MakeGolangArgs() (string, error) {
 	var args []string
 	for i, a := range obj.Args {
-		gol, err := a.ToGo()
+		gol, err := a.ToGolang()
 		if err != nil {
 			return "", err
 		}
-		args = append(args, fmt.Sprintf("input[%d].%s()", i, gol))
+		input := fmt.Sprintf("input[%d].%s()", i, gol)
+		if a.Type == "int" {
+			input = fmt.Sprintf("int(%s)", input)
+		}
+		args = append(args, input)
+	}
+	for _, a := range obj.ExtraGolangArgs {
+		args = append(args, a.Value)
 	}
 	return strings.Join(args, ", "), nil
 }
@@ -123,59 +143,39 @@ func (obj *function) Signature() (string, error) {
 
 // MakeGoReturn returns the golang signature of the return.
 func (obj *function) MakeGoReturn() (string, error) {
-	return obj.Return[0].ToGo()
+	return obj.Return[0].ToGolang()
 }
 
-// MakeGoTypeReturn returns the mcl signature of the return.
-func (obj *function) MakeGoTypeReturn() string {
-	return obj.Return[0].Type
+// ConvertStart returns the start of a casting function required to convert from mcl to golang.
+func (obj *function) ConvertStart() string {
+	t := obj.Return[0].Type
+	switch t {
+	case "int":
+		return "int64("
+	default:
+		return ""
+	}
 }
 
-// MakeTestSign returns the signature of the test.
-func (obj *function) MakeTestSign() string {
-	var args []string
-	for i, a := range obj.Args {
-		var nextSign string
-		if i+1 < len(obj.Args) {
-			nextSign = obj.Args[i+1].Type
-		} else {
-			nextSign = obj.MakeGoTypeReturn()
+// ConvertStop returns the end of the conversion function required to convert from mcl to golang.
+func (obj *function) ConvertStop() string {
+	t := obj.Return[0].Type
+	switch t {
+	case "int":
+		return ")"
+	default:
+		return ""
 	}
-		if nextSign == a.Type {
-			args = append(args, a.Name)
-		} else {
-			args = append(args, fmt.Sprintf("%s %s", a.Name, a.Type))
-		}
-	}
-	args = append(args, fmt.Sprintf("expected %s", obj.MakeGoTypeReturn()))
-	return strings.Join(args, ", ")
 }
 
-// TestInput generated a string that can be passed as test input.
-func (obj *function) TestInput() (string, error) {
-	var values []string
-	for _, i := range obj.Args {
-		tti, err := i.ToTestInput()
-		if err != nil {
-			return "", err
+// MakeGolangTypeReturn returns the mcl signature of the return.
+func (obj *function) MakeGolangTypeReturn() string {
+	t := obj.Return[0].Type
+	switch t {
+	case "int64":
+		t = "int"
+	case "float64":
+		t = "float"
 	}
-		values = append(values, tti)
-	}
-	return fmt.Sprintf("[]types.Value{%s}", strings.Join(values, ", ")), nil
-}
-
-// MakeTestArgs generates a string that can be passed a test arguments.
-func (obj *functest) MakeTestArgs() string {
-	var values []string
-	for _, i := range obj.Args {
-		if i.Type == "string" {
-			values = append(values, fmt.Sprintf(`"%s"`, i.Value))
-		}
-	}
-	for _, i := range obj.Expect {
-		if i.Type == "string" {
-			values = append(values, fmt.Sprintf(`"%s"`, i.Value))
-		}
-	}
-	return strings.Join(values, ", ")
+	return t
 }

lang/funcs/funcgen/func_test.go

@@ -0,0 +1,71 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"testing"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+func TestRenderFuncs(t *testing.T) {
+	testRenderFuncsWithFixture(t, "base")
+}
+
+func testRenderFuncsWithFixture(t *testing.T, fixture string) {
+	pkg := &golangPackage{
+		Name:    "testpkg",
+		Exclude: []string{"ToLower"},
+	}
+
+	funcs := &functions{}
+	fixtures, err := ioutil.ReadFile(fmt.Sprintf("fixtures/func_%s.yaml", fixture))
+	if err != nil {
+		t.Fatalf("Fixtures (yaml) unreadable!\n%v", err)
+	}
+	err = yaml.UnmarshalStrict(fixtures, &funcs)
+	if err != nil {
+		t.Fatalf("Fixtures (yaml) unreadable!\n%v", err)
+	}
+
+	golangFixtures, err := ioutil.ReadFile(fmt.Sprintf("fixtures/func_%s.tpl", fixture))
+	if err != nil {
+		t.Fatalf("Fixtures (tpl) unreadable!\n%v", err)
+	}
+
+	c := config{
+		Packages: []*golangPackage{pkg},
+	}
+
+	dstFileName := fmt.Sprintf("func_%s.result", fixture)
+	err = generateTemplate(c, *funcs, "fixtures", "templates/generated_funcs.go.tpl", dstFileName)
+	if err != nil {
+		t.Fatalf("Not generating template!\n%v", err)
+	}
+	result, err := ioutil.ReadFile(fmt.Sprintf("fixtures/%s", dstFileName))
+	if err != nil {
+		t.Fatalf("Result unreadable!\n%v", err)
+	}
+
+	if !reflect.DeepEqual(golangFixtures, result) {
+		t.Fatalf("Functions differ!\n1>\n%v\n2>\n%v", string(golangFixtures), string(result))
+	}
+}

lang/funcs/funcgen/pkg.go

@@ -18,13 +18,39 @@
 package main
 
 import (
+	"bytes"
+	"errors"
+	"fmt"
 	"io/ioutil"
 	"log"
+	"os/exec"
 	"path/filepath"
+	"regexp"
+	"strings"
 
+	"github.com/iancoleman/strcase"
 	yaml "gopkg.in/yaml.v2"
 )
 
+var (
+	validSignature = regexp.MustCompile(`^func (?P<name>[A-Z][a-zA-Z0-9]+)\((?P<args>([a-zA-Z]+( (string|bool|float64|int64|int))?(, )?){0,})\) (?P<return>(string|float64|int64|bool|int)|\((string|float64|int64|bool|int), error\))$`)
+	errExcluded    = errors.New("function is excluded")
+)
+
+type golangPackages []*golangPackage
+
+type golangPackage struct {
+	// Name is the name of the go package.
+	Name string `yaml:"name"`
+	// Alias is the alias of the package when imported in golang.
+	// e.g. import rand "os.rand"
+	Alias string `yaml:"alias,omitempty"`
+	// MgmtAlias is the name of the package inside mcl.
+	MgmtAlias string `yaml:"mgmtAlias,omitempty"`
+	// Exclude is a list of golang function names that we do not want.
+	Exclude []string `yaml:"exclude,omitempty"`
+}
+
 func parsePkg(path, filename, templates string) error {
 	var c config
 	filePath := filepath.Join(path, filename)
@@ -37,9 +63,173 @@ func parsePkg(path, filename, templates string) error {
 	if err != nil {
 		return err
 	}
-	err = parseFuncs(c, path, templates)
+	functions, err := parsePackages(c)
+	if err != nil {
+		return err
+	}
+	err = parseFuncs(c, functions, path, templates)
 	if err != nil {
 		return err
 	}
 	return nil
 }
+
+func parsePackages(c config) (functions, error) {
+	var funcs []function
+	for _, golangPackage := range c.Packages {
+		fn, err := golangPackage.parsefuncs()
+		if err != nil {
+			return funcs, err
+		}
+		funcs = append(funcs, fn...)
+	}
+	return funcs, nil
+}
+
+func (obj *golangPackage) parsefuncs() (functions, error) {
+	var funcs []function
+	cmd := exec.Command("go", "doc", obj.Name)
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		return funcs, err
+	}
+	return obj.extractFuncs(out.String(), true)
+}
+
+func (obj *golangPackage) extractFuncs(doc string, getHelp bool) (functions, error) {
+	var funcs []function
+	for _, line := range strings.Split(doc, "\n") {
+		if validSignature.MatchString(line) {
+			f, err := obj.parseFunctionLine(line, getHelp)
+			if err != nil && err != errExcluded {
+				return funcs, err
+			}
+			if f != nil {
+				funcs = append(funcs, *f)
+			}
+		}
+	}
+
+	return funcs, nil
+}
+
+func (obj *golangPackage) parseFunctionLine(line string, getHelp bool) (*function, error) {
+	match := validSignature.FindStringSubmatch(line)
+	result := make(map[string]string)
+	for i, name := range validSignature.SubexpNames() {
+		if i != 0 && name != "" {
+			result[name] = match[i]
+		}
+	}
+
+	name := result["name"]
+
+	for _, e := range obj.Exclude {
+		if e == name {
+			return nil, errExcluded
+		}
+	}
+
+	errorFul, err := regexp.MatchString(`, error\)$`, result["return"])
+	if err != nil {
+		return nil, err
+	}
+
+	returns := parseReturn(result["return"])
+	if len(returns) == 0 {
+		return nil, errExcluded
+	}
+
+	mgmtpackage := obj.Name
+	if obj.MgmtAlias != "" {
+		mgmtpackage = obj.MgmtAlias
+	}
+	mgmtpackage = fmt.Sprintf("golang/%s", mgmtpackage)
+
+	internalName := fmt.Sprintf("%s%s", strcase.ToCamel(strings.Replace(obj.Name, "/", "", -1)), name)
+	internalName = strings.Replace(internalName, "Html", "HTML", -1)
+	var help string
+	if getHelp {
+		help, err = obj.getHelp(name, internalName)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &function{
+		MgmtPackage:   mgmtpackage,
+		MclName:       strcase.ToSnake(name),
+		InternalName:  internalName,
+		Help:          help,
+		GolangPackage: obj,
+		GolangFunc:    name,
+		Errorful:      errorFul,
+		Args:          parseArgs(result["args"]),
+		Return:        returns,
+	}, nil
+}
+
+func reverseArgs(s []arg) {
+	last := len(s) - 1
+	for i := 0; i < len(s)/2; i++ {
+		s[i], s[last-i] = s[last-i], s[i]
+	}
+}
+func reverse(s []string) {
+	last := len(s) - 1
+	for i := 0; i < len(s)/2; i++ {
+		s[i], s[last-i] = s[last-i], s[i]
+	}
+}
+
+func parseArgs(str string) []arg {
+	var args []arg
+	s := strings.Split(str, ",")
+	reverse(s)
+	var currentType string
+	for _, currentArg := range s {
+		if currentArg == "" {
+			continue
+		}
+		v := strings.Split(strings.TrimSpace(currentArg), " ")
+		if len(v) == 2 {
+			currentType = v[1]
+		}
+		args = append(args, arg{Name: v[0], Type: currentType})
+	}
+	reverseArgs(args)
+	return args
+}
+
+func parseReturn(str string) []arg {
+	var returns []arg
+	re := regexp.MustCompile(`(int64|float64|string|bool|int)`)
+	t := string(re.Find([]byte(str)))
+	returns = append(returns, arg{Type: t})
+	return returns
+}
+
+func (obj *golangPackage) getHelp(function, internalName string) (string, error) {
+	cmd := exec.Command("go", "doc", fmt.Sprintf("%s.%s", obj.Name, function))
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		return "", err
+	}
+	var doc string
+	for i, line := range strings.Split(strings.TrimSpace(out.String()), "\n") {
+		if i > 0 {
+			s := strings.TrimSpace(line)
+			if i == 1 {
+				docs := strings.Split(s, " ")
+				docs[0] = internalName
+				s = strings.Join(docs, " ")
+			}
+			doc = fmt.Sprintf("%s// %s is an autogenerated function.\n", doc, s)
+		}
+	}
+	return doc, nil
+}

lang/funcs/funcgen/pkg_test.go

@@ -0,0 +1,61 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"testing"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+func TestParseFuncs(t *testing.T) {
+	testParseFuncsWithFixture(t, "base")
+}
+
+func testParseFuncsWithFixture(t *testing.T, fixture string) {
+	pkg := &golangPackage{
+		Name:    "testpkg",
+		Exclude: []string{"ToLower"},
+	}
+
+	signatures, err := ioutil.ReadFile(fmt.Sprintf("fixtures/func_%s.txt", fixture))
+	if err != nil {
+		t.Fatalf("Fixtures (txt) unreadable!\n%v", err)
+	}
+	f, err := pkg.extractFuncs(string(signatures), false)
+	if err != nil {
+		t.Fatalf("Error while parsing functions: %v", err)
+	}
+
+	expected := &functions{}
+	fixtures, err := ioutil.ReadFile(fmt.Sprintf("fixtures/func_%s.yaml", fixture))
+	if err != nil {
+		t.Fatalf("Fixtures (yaml) unreadable!\n%v", err)
+	}
+	err = yaml.UnmarshalStrict(fixtures, &expected)
+	if err != nil {
+		t.Fatalf("Fixtures (yaml) unreadable!\n%v", err)
+	}
+
+	if !reflect.DeepEqual(f, *expected) {
+		t.Fatalf("Functions differ!\n%v\n%v", f, *expected)
+	}
+}

lang/funcs/funcgen/templates/generated_funcs.go.tpl

@@ -18,24 +18,33 @@
 package core
 
 import (
-	"strings"
-
+{{ range $i, $func := .Packages }}	{{ if not (eq .Alias "") }}{{.Alias}} {{end}}"{{.Name}}"
+{{ end }}
 	"github.com/purpleidea/mgmt/lang/funcs/simple"
 	"github.com/purpleidea/mgmt/lang/types"
 )
 
 func init() {
-{{ range $i, $func := .Functions }}	simple.ModuleRegister("{{$func.MgmtPackage}}", "{{$func.MgmtName}}", &types.FuncValue{
+{{ range $i, $func := .Functions }}	simple.ModuleRegister("{{$func.MgmtPackage}}", "{{$func.MclName}}", &types.FuncValue{
 		T: types.NewType("{{$func.Signature}}"),
-		V: {{$func.GoFunc}},
+		V: {{$func.InternalName}},
 	})
 {{ end }}
 }
 {{ range $i, $func := .Functions }}
-// {{$func.GoFunc}} {{$func.Help}}
-func {{$func.GoFunc}}(input []types.Value) (types.Value, error) {
+{{$func.Help}}func {{$func.InternalName}}(input []types.Value) (types.Value, error) {
+{{- if $func.Errorful }}
+	v, err := {{ if not (eq $func.GolangPackage.Alias "") }}{{$func.GolangPackage.Alias}}{{else}}{{$func.GolangPackage.Name}}{{end}}.{{$func.GolangFunc}}({{$func.MakeGolangArgs}})
+	if err != nil {
+		return nil, err
+	}
 	return &types.{{$func.MakeGoReturn}}Value{
-		V: {{$func.GoPackage}}.{{$func.GoFunc}}({{$func.MakeGoArgs}}),
+		V: {{$func.ConvertStart}}v{{$func.ConvertStop}},
 	}, nil
+{{ else }}
+	return &types.{{$func.MakeGoReturn}}Value{
+		V: {{$func.ConvertStart}}{{ if not (eq $func.GolangPackage.Alias "") }}{{$func.GolangPackage.Alias}}{{else}}{{$func.GolangPackage.Name}}{{end}}.{{$func.GolangFunc}}({{$func.MakeGolangArgs}}{{$func.ConvertStop}}),
+	}, nil
+{{ end -}}
 }
 {{ end -}}

lang/funcs/funcgen/templates/generated_funcs_test.go.tpl

@@ -1,41 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2019+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package core
-
-import (
-	"testing"
-
-	"github.com/purpleidea/mgmt/lang/types"
-)
-{{ range $i, $func := .Functions }}
-func test{{$func.GoFunc}}(t *testing.T, {{$func.MakeTestSign}}) {
-	value, err := {{$func.GoFunc}}({{$func.TestInput}})
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	if value.{{$func.MakeGoReturn}}() != expected {
-		t.Errorf("invalid output, expected %s, got %s", expected, value.{{$func.MakeGoReturn}}())
-	}
-}
-{{ range $index, $test := $func.Tests }}
-func Test{{$func.GoFunc}}{{$index}}(t *testing.T) {
-	test{{$func.GoFunc}}(t, {{.MakeTestArgs}})
-}
-{{ end -}}
-{{ end -}}

lang/interpret_test/TestAstFunc2/deploy-readfile0/main.mcl

@@ -1,4 +1,4 @@
-import "strings"
+import "golang/strings"
 import "deploy"
 import "second.mcl"
 import "mod1/"

lang/interpret_test/TestAstFunc2/deploy-readfile1/main.mcl

@@ -1,4 +1,4 @@
-import "strings"
+import "golang/strings"
 import "deploy"
 import "second.mcl"
 import "mod1/"