diff --git a/Makefile b/Makefile index 24061990..4904317f 100644 --- a/Makefile +++ b/Makefile @@ -504,14 +504,10 @@ help: ## show this help screen awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' @echo '' -funcgen: lang/funcs/core/generated_funcs_test.go lang/funcs/core/generated_funcs.go - -lang/funcs/core/generated_funcs_test.go: lang/funcs/funcgen/*.go lang/funcs/core/funcgen.yaml lang/funcs/funcgen/templates/generated_funcs_test.go.tpl - @echo "Generating: funcs test..." - @go run lang/funcs/funcgen/*.go -templates lang/funcs/funcgen/templates/generated_funcs_test.go.tpl 2>/dev/null +funcgen: lang/funcs/core/generated_funcs.go lang/funcs/core/generated_funcs.go: lang/funcs/funcgen/*.go lang/funcs/core/funcgen.yaml lang/funcs/funcgen/templates/generated_funcs.go.tpl @echo "Generating: funcs..." - @go run lang/funcs/funcgen/*.go -templates lang/funcs/funcgen/templates/generated_funcs.go.tpl 2>/dev/null + @go run `find lang/funcs/funcgen/ -maxdepth 1 -type f -name '*.go' -not -name '*_test.go'` -templates=lang/funcs/funcgen/templates/generated_funcs.go.tpl 2>/dev/null # vim: ts=8 diff --git a/examples/lang/html.mcl b/examples/lang/html.mcl new file mode 100644 index 00000000..e71cfc71 --- /dev/null +++ b/examples/lang/html.mcl @@ -0,0 +1,10 @@ +import "golang/html" +import "fmt" + +$text1 = html.unescape_string("<h1>MGMT!</h1>") +$text2 = html.escape_string("Test & Re-Test\n") + +file "/tmp/index.html" { + state => "exists", + content => "${text1}${text2}", +} diff --git a/examples/lang/os.mcl b/examples/lang/os.mcl new file mode 100644 index 00000000..937189c7 --- /dev/null +++ b/examples/lang/os.mcl @@ -0,0 +1,27 @@ +import "golang/os" +import "golang/exec" +import "fmt" + +$tmpdir = os.temp_dir() + +file "${tmpdir}/execinfo" { + state => "exists", + content => fmt.printf("mgmt is at %s\n", os.executable()), +} + +file "${tmpdir}/mgmtenv" { + state => "exists", + content => os.expand_env("$HOME sweet ${os.getenv(\"HOME\")}\n"), +} + +file "${tmpdir}/mgmtos" { + state => "exists", + content => os.readlink("/bin"), +} + +$rm = exec.look_path("rm") + +file "${tmpdir}/cache" { + state => "exists", + content => "Plz cache in ${os.user_cache_dir()}.\nYour home is ${os.user_home_dir()}. Remove with ${rm}\n", +} diff --git a/examples/lang/runtime.mcl b/examples/lang/runtime.mcl new file mode 100644 index 00000000..273e9cd0 --- /dev/null +++ b/examples/lang/runtime.mcl @@ -0,0 +1,7 @@ +import "golang/runtime" +import "fmt" + +file "/tmp/mgmtinfo" { + state => "exists", + content => fmt.printf("Hi from mgmt! mgmt is running with %s; and GOROOT is %s.\n", runtime.version(), runtime.goroot()), +} diff --git a/lang/funcs/core/funcgen.yaml b/lang/funcs/core/funcgen.yaml index 1978c4b2..4c616a7e 100644 --- a/lang/funcs/core/funcgen.yaml +++ b/lang/funcs/core/funcgen.yaml @@ -1,55 +1,19 @@ # This file is used by github.com/purpleidea/mgmt/lang/funcs/funcgen/ to # generate mcl functions. -functions: -- mgmtName: to_upper - mgmtPackage: strings - help: turns a string to uppercase. - goPackage: strings - goFunc: ToUpper - args: [{name: a, type: string}] - return: [{type: string}] - tests: - - args: [{type: string, value: "Hello"}] - return: [{type: string, value: "HELLO"}] - - args: [{type: string, value: "HELLO 22"}] - return: [{type: string, value: "HELLO 22"}] -- mgmtName: trim - mgmtPackage: strings - help: returns a slice of the string s with all leading and trailing Unicode code points contained in cutset removed. - goPackage: strings - goFunc: Trim - args: [{name: s, type: string}, {name: cutset, type: string}] - return: [{type: string}] - tests: - - args: [{type: string, value: "??Hello.."}, {type: string, value: "?."}] - return: [{type: string, value: "Hello"}] -- mgmtName: trim_left - mgmtPackage: strings - help: returns a slice of the string s with all leading Unicode code points contained in cutset removed. - goPackage: strings - goFunc: TrimLeft - args: [{name: s, type: string}, {name: cutset, type: string}] - return: [{type: string}] - tests: - - args: [{type: string, value: "??Hello.."}, {type: string, value: "?."}] - return: [{type: string, value: "Hello.."}] -- mgmtName: trim_space - mgmtPackage: strings - help: returns a slice of the string s, with all leading and trailing white space removed, as defined by Unicode. - goPackage: strings - goFunc: TrimSpace - args: [{name: s, type: string}] - return: [{type: string}] - tests: - - args: [{type: string, value: "Hello 2 "}] - return: [{type: string, value: "Hello 2"}] -- mgmtName: trim_right - mgmtPackage: strings - help: returns a slice of the string s with all trailing Unicode code points contained in cutset removed. - goPackage: strings - goFunc: TrimRight - args: [{name: s, type: string}, {name: cutset, type: string}] - return: [{type: string}] - tests: - - args: [{type: string, value: "??Hello.."}, {type: string, value: "?."}] - return: [{type: string, value: "??Hello"}] +packages: +- name: html +- name: math +- name: math/rand + alias: rand + mgmtAlias: rand +- name: os +- name: os/exec + alias: exec + mgmtAlias: exec +- name: path +- name: path/filepath + alias: filepath + mgmtAlias: filepath +- name: runtime +- name: strconv +- name: strings diff --git a/lang/funcs/funcgen/config.go b/lang/funcs/funcgen/config.go index a2091a35..291d3d76 100644 --- a/lang/funcs/funcgen/config.go +++ b/lang/funcs/funcgen/config.go @@ -24,45 +24,78 @@ import ( ) type config struct { - Functions functions `yaml:"functions"` + Packages golangPackages `yaml:"packages"` } type functions []function -type testarg struct { - Name string `yaml:"name,omitempty"` - Type string `yaml:"type"` - Value string `yaml:"value"` +type arg struct { + // Name is the name of the argument. + Name string `yaml:"name,omitempty"` + // Value is the value of the argument. + Value string `yaml:"value,omitempty"` + // Type is the type of the argument. + // Supported: bool, string, int, int64, float64. + Type string `yaml:"type"` } -type arg struct { - Name string `yaml:"name,omitempty"` - Type string `yaml:"type"` +// GolangType prints the golang equivalent of a mcl type. +func (obj *arg) GolangType() string { + t := obj.Type + if t == "float" { + return "float64" + } + return t } // ToMcl prints the arg signature as expected by mcl. func (obj *arg) ToMcl() (string, error) { - if obj.Type == "string" { - if obj.Name != "" { - return fmt.Sprintf("%s str", obj.Name), nil - } - return types.TypeStr.String(), nil + var prefix string + if obj.Name != "" { + prefix = fmt.Sprintf("%s ", obj.Name) + } + switch obj.Type { + case "bool": + return fmt.Sprintf("%s%s", prefix, types.TypeBool.String()), nil + case "string": + return fmt.Sprintf("%s%s", prefix, types.TypeStr.String()), nil + case "int64", "int": + return fmt.Sprintf("%s%s", prefix, types.TypeInt.String()), nil + case "float64": + return fmt.Sprintf("%s%s", prefix, types.TypeFloat.String()), nil + default: + return "", fmt.Errorf("cannot convert %v to mcl", obj) } - return "", fmt.Errorf("cannot convert %v to mcl", obj) } // ToGo prints the arg signature as expected by golang. -func (obj *arg) ToGo() (string, error) { - if obj.Type == "string" { +func (obj *arg) ToGolang() (string, error) { + switch obj.Type { + case "bool": + return "Bool", nil + case "string": return "Str", nil + case "int", "int64": + return "Int", nil + case "float64": + return "Float", nil + default: + return "", fmt.Errorf("cannot convert %v to golang", obj) } - return "", fmt.Errorf("cannot convert %v to go", obj) } // ToTestInput prints the arg signature as expected by tests. func (obj *arg) ToTestInput() (string, error) { - if obj.Type == "string" { + switch obj.Type { + case "bool": + return fmt.Sprintf("&types.BoolValue{V: %s}", obj.Name), nil + case "string": return fmt.Sprintf("&types.StrValue{V: %s}", obj.Name), nil + case "int": + return fmt.Sprintf("&types.IntValue{V: %s}", obj.Name), nil + case "float": + return fmt.Sprintf("&types.FloatValue{V: %s}", obj.Name), nil + default: + return "", fmt.Errorf("cannot convert %v to test input", obj) } - return "", fmt.Errorf("cannot convert %v to test input", obj) } diff --git a/lang/funcs/funcgen/fixtures/.gitignore b/lang/funcs/funcgen/fixtures/.gitignore new file mode 100644 index 00000000..aedae76e --- /dev/null +++ b/lang/funcs/funcgen/fixtures/.gitignore @@ -0,0 +1 @@ +*.result diff --git a/lang/funcs/funcgen/fixtures/func_base.tpl b/lang/funcs/funcgen/fixtures/func_base.tpl new file mode 100644 index 00000000..51135acd --- /dev/null +++ b/lang/funcs/funcgen/fixtures/func_base.tpl @@ -0,0 +1,83 @@ +// Mgmt +// Copyright (C) 2013-2019+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package core + +import ( + "testpkg" + + "github.com/purpleidea/mgmt/lang/funcs/simple" + "github.com/purpleidea/mgmt/lang/types" +) + +func init() { + simple.ModuleRegister("golang/testpkg", "all_kind", &types.FuncValue{ + T: types.NewType("func(x int, y str) float"), + V: TestpkgAllKind, + }) + simple.ModuleRegister("golang/testpkg", "to_upper", &types.FuncValue{ + T: types.NewType("func(s str) str"), + V: TestpkgToUpper, + }) + simple.ModuleRegister("golang/testpkg", "max", &types.FuncValue{ + T: types.NewType("func(x float, y float) float"), + V: TestpkgMax, + }) + simple.ModuleRegister("golang/testpkg", "with_error", &types.FuncValue{ + T: types.NewType("func(s str) str"), + V: TestpkgWithError, + }) + simple.ModuleRegister("golang/testpkg", "with_int", &types.FuncValue{ + T: types.NewType("func(s float, i int, x int, j int, k int, b bool, t str) str"), + V: TestpkgWithInt, + }) + +} + +func TestpkgAllKind(input []types.Value) (types.Value, error) { + return &types.FloatValue{ + V: testpkg.AllKind(input[0].Int(), input[1].Str()), + }, nil +} + +func TestpkgToUpper(input []types.Value) (types.Value, error) { + return &types.StrValue{ + V: testpkg.ToUpper(input[0].Str()), + }, nil +} + +func TestpkgMax(input []types.Value) (types.Value, error) { + return &types.FloatValue{ + V: testpkg.Max(input[0].Float(), input[1].Float()), + }, nil +} + +func TestpkgWithError(input []types.Value) (types.Value, error) { + v, err := testpkg.WithError(input[0].Str()) + if err != nil { + return nil, err + } + return &types.StrValue{ + V: v, + }, nil +} + +func TestpkgWithInt(input []types.Value) (types.Value, error) { + return &types.StrValue{ + V: testpkg.WithInt(input[0].Float(), int(input[1].Int()), input[2].Int(), int(input[3].Int()), int(input[4].Int()), input[5].Bool(), input[6].Str()), + }, nil +} diff --git a/lang/funcs/funcgen/fixtures/func_base.txt b/lang/funcs/funcgen/fixtures/func_base.txt new file mode 100644 index 00000000..70b58dee --- /dev/null +++ b/lang/funcs/funcgen/fixtures/func_base.txt @@ -0,0 +1,14 @@ +Random +const E = 2.71828182845904523536028747135266249775724709369995957496696763 ... +const MaxFloat32 = 3.40282346638528859811704183484516925440e+38 ... +const MaxInt8 = 1<<7 - 1 ... +func Lgamma(x float64) (lgamma float64, sign int) +func AllKind(x int64, y string) float64 +func ToUpper(s string) string +func ToLower(s string) string +func Max(x, y float64) float64 +func WithError(s string) (string, error) +func WithErrorButNothingElse(s string) error +func WithNothingElse(s string) +func Nextafter32(x, y float32) (r float32) +func WithInt(s float64, i int, x int64, j, k int, b bool, t string) string diff --git a/lang/funcs/funcgen/fixtures/func_base.yaml b/lang/funcs/funcgen/fixtures/func_base.yaml new file mode 100644 index 00000000..f549cc8c --- /dev/null +++ b/lang/funcs/funcgen/fixtures/func_base.yaml @@ -0,0 +1,42 @@ +- mgmtPackage: golang/testpkg + mgmtName: all_kind + internalName: TestpkgAllKind + golangPackage: &pkg + name: testpkg + exclude: [ToLower] + golangFunc: AllKind + errorful: false + args: [{name: x, type: int64},{name: y, type: string}] + return: [{type: float64}] +- mgmtPackage: golang/testpkg + mgmtName: to_upper + internalName: TestpkgToUpper + golangPackage: *pkg + golangFunc: ToUpper + errorful: false + args: [{name: s, type: string}] + return: [{type: string}] +- mgmtPackage: golang/testpkg + mgmtName: max + internalName: TestpkgMax + golangPackage: *pkg + golangFunc: Max + errorful: false + args: [{name: x, type: float64},{name: y, type: float64}] + return: [{type: float64}] +- mgmtPackage: golang/testpkg + mgmtName: with_error + internalName: TestpkgWithError + golangPackage: *pkg + golangFunc: WithError + errorful: true + args: [{name: s, type: string}] + return: [{type: string}] +- mgmtPackage: golang/testpkg + mgmtName: with_int + internalName: TestpkgWithInt + golangPackage: *pkg + golangFunc: WithInt + errorful: false + args: [{name: s, type: float64}, {name: i, type: int}, {name: x, type: int64}, {name: j, type: int}, {name: k, type: int}, {name: b, type: bool}, {name: t, type: string}] + return: [{type: string}] diff --git a/lang/funcs/funcgen/func.go b/lang/funcs/funcgen/func.go index 5d569588..8b20e796 100644 --- a/lang/funcs/funcgen/func.go +++ b/lang/funcs/funcgen/func.go @@ -28,34 +28,40 @@ import ( ) type function struct { - MgmtPackage string `yaml:"mgmtPackage"` - MgmtName string `yaml:"mgmtName"` - Help string `yaml:"help"` - GoPackage string `yaml:"goPackage"` - GoFunc string `yaml:"goFunc"` - Args []arg `yaml:"args"` - Return []arg `yaml:"return"` - Tests []functest `yaml:"tests"` + // MclName is the name of the package of the function in mcl. + MgmtPackage string `yaml:"mgmtPackage"` + // MclName is the name of the function in mcl. + MclName string `yaml:"mgmtName"` + // InternalName is the name used inside the templated file. + // Used to avoid clash between same functions from different packages. + InternalName string `yaml:"internalName"` + // Help is the docstring of the function, including // and + // new lines. + Help string `yaml:"help"` + // GolangPackage is the representation of the package. + GolangPackage *golangPackage `yaml:"golangPackage"` + // GolangFunc is the name of the function in golang. + GolangFunc string `yaml:"golangFunc"` + // Errorful indicates wether the golang function can return an error + // as second argument. + Errorful bool `yaml:"errorful"` + // Args is the list of the arguments of the function. + Args []arg `yaml:"args"` + // ExtraGolangArgs are arguments that are added at the end of the go call. + // e.g. strconv.ParseFloat("3.1415", 64) could require add 64. + ExtraGolangArgs []arg `yaml:"extraGolangArgs"` + // Return is the list of arguments returned by the function. + Return []arg `yaml:"return"` } -type functest struct { - Args []testarg `yaml:"args"` - Expect []testarg `yaml:"return"` -} - -type templateInput struct { - Func function - MgmtPackage string -} - -func parseFuncs(c config, path, templates string) error { +func parseFuncs(c config, f functions, path, templates string) error { templateFiles, err := filepath.Glob(templates) if err != nil { return err } for _, tpl := range templateFiles { log.Printf("Template: %s", tpl) - err = generateTemplate(c, path, tpl) + err = generateTemplate(c, f, path, tpl, "") if err != nil { return err } @@ -63,7 +69,7 @@ func parseFuncs(c config, path, templates string) error { return nil } -func generateTemplate(c config, path, templateFile string) error { +func generateTemplate(c config, f functions, path, templateFile, finalName string) error { log.Printf("Reading: %s", templateFile) basename := filepath.Base(templateFile) tplFile, err := ioutil.ReadFile(templateFile) @@ -74,28 +80,42 @@ func generateTemplate(c config, path, templateFile string) error { if err != nil { return err } - finalName := strings.TrimSuffix(basename, ".tpl") + if finalName == "" { + finalName = strings.TrimSuffix(basename, ".tpl") + } finalPath := filepath.Join(path, finalName) - log.Printf("Writing: %s", finalPath) finalFile, err := os.Create(finalPath) if err != nil { return err } - if err = t.Execute(finalFile, c); err != nil { + if err = t.Execute(finalFile, struct { + Packages golangPackages + Functions []function + }{ + c.Packages, + f, + }); err != nil { return err } return nil } -// MakeGoArgs translates the func args to go args. -func (obj *function) MakeGoArgs() (string, error) { +// MakeGolangArgs translates the func args to golang args. +func (obj *function) MakeGolangArgs() (string, error) { var args []string for i, a := range obj.Args { - gol, err := a.ToGo() + gol, err := a.ToGolang() if err != nil { return "", err } - args = append(args, fmt.Sprintf("input[%d].%s()", i, gol)) + input := fmt.Sprintf("input[%d].%s()", i, gol) + if a.Type == "int" { + input = fmt.Sprintf("int(%s)", input) + } + args = append(args, input) + } + for _, a := range obj.ExtraGolangArgs { + args = append(args, a.Value) } return strings.Join(args, ", "), nil } @@ -123,59 +143,39 @@ func (obj *function) Signature() (string, error) { // MakeGoReturn returns the golang signature of the return. func (obj *function) MakeGoReturn() (string, error) { - return obj.Return[0].ToGo() + return obj.Return[0].ToGolang() } -// MakeGoTypeReturn returns the mcl signature of the return. -func (obj *function) MakeGoTypeReturn() string { - return obj.Return[0].Type +// ConvertStart returns the start of a casting function required to convert from mcl to golang. +func (obj *function) ConvertStart() string { + t := obj.Return[0].Type + switch t { + case "int": + return "int64(" + default: + return "" + } } -// MakeTestSign returns the signature of the test. -func (obj *function) MakeTestSign() string { - var args []string - for i, a := range obj.Args { - var nextSign string - if i+1 < len(obj.Args) { - nextSign = obj.Args[i+1].Type - } else { - nextSign = obj.MakeGoTypeReturn() - } - if nextSign == a.Type { - args = append(args, a.Name) - } else { - args = append(args, fmt.Sprintf("%s %s", a.Name, a.Type)) - } +// ConvertStop returns the end of the conversion function required to convert from mcl to golang. +func (obj *function) ConvertStop() string { + t := obj.Return[0].Type + switch t { + case "int": + return ")" + default: + return "" } - args = append(args, fmt.Sprintf("expected %s", obj.MakeGoTypeReturn())) - return strings.Join(args, ", ") } -// TestInput generated a string that can be passed as test input. -func (obj *function) TestInput() (string, error) { - var values []string - for _, i := range obj.Args { - tti, err := i.ToTestInput() - if err != nil { - return "", err - } - values = append(values, tti) +// MakeGolangTypeReturn returns the mcl signature of the return. +func (obj *function) MakeGolangTypeReturn() string { + t := obj.Return[0].Type + switch t { + case "int64": + t = "int" + case "float64": + t = "float" } - return fmt.Sprintf("[]types.Value{%s}", strings.Join(values, ", ")), nil -} - -// MakeTestArgs generates a string that can be passed a test arguments. -func (obj *functest) MakeTestArgs() string { - var values []string - for _, i := range obj.Args { - if i.Type == "string" { - values = append(values, fmt.Sprintf(`"%s"`, i.Value)) - } - } - for _, i := range obj.Expect { - if i.Type == "string" { - values = append(values, fmt.Sprintf(`"%s"`, i.Value)) - } - } - return strings.Join(values, ", ") + return t } diff --git a/lang/funcs/funcgen/func_test.go b/lang/funcs/funcgen/func_test.go new file mode 100644 index 00000000..3be01d1e --- /dev/null +++ b/lang/funcs/funcgen/func_test.go @@ -0,0 +1,71 @@ +// Mgmt +// Copyright (C) 2013-2019+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package main + +import ( + "fmt" + "io/ioutil" + "reflect" + "testing" + + yaml "gopkg.in/yaml.v2" +) + +func TestRenderFuncs(t *testing.T) { + testRenderFuncsWithFixture(t, "base") +} + +func testRenderFuncsWithFixture(t *testing.T, fixture string) { + pkg := &golangPackage{ + Name: "testpkg", + Exclude: []string{"ToLower"}, + } + + funcs := &functions{} + fixtures, err := ioutil.ReadFile(fmt.Sprintf("fixtures/func_%s.yaml", fixture)) + if err != nil { + t.Fatalf("Fixtures (yaml) unreadable!\n%v", err) + } + err = yaml.UnmarshalStrict(fixtures, &funcs) + if err != nil { + t.Fatalf("Fixtures (yaml) unreadable!\n%v", err) + } + + golangFixtures, err := ioutil.ReadFile(fmt.Sprintf("fixtures/func_%s.tpl", fixture)) + if err != nil { + t.Fatalf("Fixtures (tpl) unreadable!\n%v", err) + } + + c := config{ + Packages: []*golangPackage{pkg}, + } + + dstFileName := fmt.Sprintf("func_%s.result", fixture) + err = generateTemplate(c, *funcs, "fixtures", "templates/generated_funcs.go.tpl", dstFileName) + if err != nil { + t.Fatalf("Not generating template!\n%v", err) + } + result, err := ioutil.ReadFile(fmt.Sprintf("fixtures/%s", dstFileName)) + if err != nil { + t.Fatalf("Result unreadable!\n%v", err) + } + + if !reflect.DeepEqual(golangFixtures, result) { + t.Fatalf("Functions differ!\n1>\n%v\n2>\n%v", string(golangFixtures), string(result)) + } +} diff --git a/lang/funcs/funcgen/pkg.go b/lang/funcs/funcgen/pkg.go index 5c3b8494..abeab790 100644 --- a/lang/funcs/funcgen/pkg.go +++ b/lang/funcs/funcgen/pkg.go @@ -18,13 +18,39 @@ package main import ( + "bytes" + "errors" + "fmt" "io/ioutil" "log" + "os/exec" "path/filepath" + "regexp" + "strings" + "github.com/iancoleman/strcase" yaml "gopkg.in/yaml.v2" ) +var ( + validSignature = regexp.MustCompile(`^func (?P[A-Z][a-zA-Z0-9]+)\((?P([a-zA-Z]+( (string|bool|float64|int64|int))?(, )?){0,})\) (?P(string|float64|int64|bool|int)|\((string|float64|int64|bool|int), error\))$`) + errExcluded = errors.New("function is excluded") +) + +type golangPackages []*golangPackage + +type golangPackage struct { + // Name is the name of the go package. + Name string `yaml:"name"` + // Alias is the alias of the package when imported in golang. + // e.g. import rand "os.rand" + Alias string `yaml:"alias,omitempty"` + // MgmtAlias is the name of the package inside mcl. + MgmtAlias string `yaml:"mgmtAlias,omitempty"` + // Exclude is a list of golang function names that we do not want. + Exclude []string `yaml:"exclude,omitempty"` +} + func parsePkg(path, filename, templates string) error { var c config filePath := filepath.Join(path, filename) @@ -37,9 +63,173 @@ func parsePkg(path, filename, templates string) error { if err != nil { return err } - err = parseFuncs(c, path, templates) + functions, err := parsePackages(c) + if err != nil { + return err + } + err = parseFuncs(c, functions, path, templates) if err != nil { return err } return nil } + +func parsePackages(c config) (functions, error) { + var funcs []function + for _, golangPackage := range c.Packages { + fn, err := golangPackage.parsefuncs() + if err != nil { + return funcs, err + } + funcs = append(funcs, fn...) + } + return funcs, nil +} + +func (obj *golangPackage) parsefuncs() (functions, error) { + var funcs []function + cmd := exec.Command("go", "doc", obj.Name) + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + return funcs, err + } + return obj.extractFuncs(out.String(), true) +} + +func (obj *golangPackage) extractFuncs(doc string, getHelp bool) (functions, error) { + var funcs []function + for _, line := range strings.Split(doc, "\n") { + if validSignature.MatchString(line) { + f, err := obj.parseFunctionLine(line, getHelp) + if err != nil && err != errExcluded { + return funcs, err + } + if f != nil { + funcs = append(funcs, *f) + } + } + } + + return funcs, nil +} + +func (obj *golangPackage) parseFunctionLine(line string, getHelp bool) (*function, error) { + match := validSignature.FindStringSubmatch(line) + result := make(map[string]string) + for i, name := range validSignature.SubexpNames() { + if i != 0 && name != "" { + result[name] = match[i] + } + } + + name := result["name"] + + for _, e := range obj.Exclude { + if e == name { + return nil, errExcluded + } + } + + errorFul, err := regexp.MatchString(`, error\)$`, result["return"]) + if err != nil { + return nil, err + } + + returns := parseReturn(result["return"]) + if len(returns) == 0 { + return nil, errExcluded + } + + mgmtpackage := obj.Name + if obj.MgmtAlias != "" { + mgmtpackage = obj.MgmtAlias + } + mgmtpackage = fmt.Sprintf("golang/%s", mgmtpackage) + + internalName := fmt.Sprintf("%s%s", strcase.ToCamel(strings.Replace(obj.Name, "/", "", -1)), name) + internalName = strings.Replace(internalName, "Html", "HTML", -1) + var help string + if getHelp { + help, err = obj.getHelp(name, internalName) + if err != nil { + return nil, err + } + } + + return &function{ + MgmtPackage: mgmtpackage, + MclName: strcase.ToSnake(name), + InternalName: internalName, + Help: help, + GolangPackage: obj, + GolangFunc: name, + Errorful: errorFul, + Args: parseArgs(result["args"]), + Return: returns, + }, nil +} + +func reverseArgs(s []arg) { + last := len(s) - 1 + for i := 0; i < len(s)/2; i++ { + s[i], s[last-i] = s[last-i], s[i] + } +} +func reverse(s []string) { + last := len(s) - 1 + for i := 0; i < len(s)/2; i++ { + s[i], s[last-i] = s[last-i], s[i] + } +} + +func parseArgs(str string) []arg { + var args []arg + s := strings.Split(str, ",") + reverse(s) + var currentType string + for _, currentArg := range s { + if currentArg == "" { + continue + } + v := strings.Split(strings.TrimSpace(currentArg), " ") + if len(v) == 2 { + currentType = v[1] + } + args = append(args, arg{Name: v[0], Type: currentType}) + } + reverseArgs(args) + return args +} + +func parseReturn(str string) []arg { + var returns []arg + re := regexp.MustCompile(`(int64|float64|string|bool|int)`) + t := string(re.Find([]byte(str))) + returns = append(returns, arg{Type: t}) + return returns +} + +func (obj *golangPackage) getHelp(function, internalName string) (string, error) { + cmd := exec.Command("go", "doc", fmt.Sprintf("%s.%s", obj.Name, function)) + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + return "", err + } + var doc string + for i, line := range strings.Split(strings.TrimSpace(out.String()), "\n") { + if i > 0 { + s := strings.TrimSpace(line) + if i == 1 { + docs := strings.Split(s, " ") + docs[0] = internalName + s = strings.Join(docs, " ") + } + doc = fmt.Sprintf("%s// %s is an autogenerated function.\n", doc, s) + } + } + return doc, nil +} diff --git a/lang/funcs/funcgen/pkg_test.go b/lang/funcs/funcgen/pkg_test.go new file mode 100644 index 00000000..afebfa92 --- /dev/null +++ b/lang/funcs/funcgen/pkg_test.go @@ -0,0 +1,61 @@ +// Mgmt +// Copyright (C) 2013-2019+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package main + +import ( + "fmt" + "io/ioutil" + "reflect" + "testing" + + yaml "gopkg.in/yaml.v2" +) + +func TestParseFuncs(t *testing.T) { + testParseFuncsWithFixture(t, "base") +} + +func testParseFuncsWithFixture(t *testing.T, fixture string) { + pkg := &golangPackage{ + Name: "testpkg", + Exclude: []string{"ToLower"}, + } + + signatures, err := ioutil.ReadFile(fmt.Sprintf("fixtures/func_%s.txt", fixture)) + if err != nil { + t.Fatalf("Fixtures (txt) unreadable!\n%v", err) + } + f, err := pkg.extractFuncs(string(signatures), false) + if err != nil { + t.Fatalf("Error while parsing functions: %v", err) + } + + expected := &functions{} + fixtures, err := ioutil.ReadFile(fmt.Sprintf("fixtures/func_%s.yaml", fixture)) + if err != nil { + t.Fatalf("Fixtures (yaml) unreadable!\n%v", err) + } + err = yaml.UnmarshalStrict(fixtures, &expected) + if err != nil { + t.Fatalf("Fixtures (yaml) unreadable!\n%v", err) + } + + if !reflect.DeepEqual(f, *expected) { + t.Fatalf("Functions differ!\n%v\n%v", f, *expected) + } +} diff --git a/lang/funcs/funcgen/templates/generated_funcs.go.tpl b/lang/funcs/funcgen/templates/generated_funcs.go.tpl index 6c512bd4..643c05c5 100644 --- a/lang/funcs/funcgen/templates/generated_funcs.go.tpl +++ b/lang/funcs/funcgen/templates/generated_funcs.go.tpl @@ -18,24 +18,33 @@ package core import ( - "strings" - +{{ range $i, $func := .Packages }} {{ if not (eq .Alias "") }}{{.Alias}} {{end}}"{{.Name}}" +{{ end }} "github.com/purpleidea/mgmt/lang/funcs/simple" "github.com/purpleidea/mgmt/lang/types" ) func init() { -{{ range $i, $func := .Functions }} simple.ModuleRegister("{{$func.MgmtPackage}}", "{{$func.MgmtName}}", &types.FuncValue{ +{{ range $i, $func := .Functions }} simple.ModuleRegister("{{$func.MgmtPackage}}", "{{$func.MclName}}", &types.FuncValue{ T: types.NewType("{{$func.Signature}}"), - V: {{$func.GoFunc}}, + V: {{$func.InternalName}}, }) {{ end }} } {{ range $i, $func := .Functions }} -// {{$func.GoFunc}} {{$func.Help}} -func {{$func.GoFunc}}(input []types.Value) (types.Value, error) { +{{$func.Help}}func {{$func.InternalName}}(input []types.Value) (types.Value, error) { +{{- if $func.Errorful }} + v, err := {{ if not (eq $func.GolangPackage.Alias "") }}{{$func.GolangPackage.Alias}}{{else}}{{$func.GolangPackage.Name}}{{end}}.{{$func.GolangFunc}}({{$func.MakeGolangArgs}}) + if err != nil { + return nil, err + } return &types.{{$func.MakeGoReturn}}Value{ - V: {{$func.GoPackage}}.{{$func.GoFunc}}({{$func.MakeGoArgs}}), + V: {{$func.ConvertStart}}v{{$func.ConvertStop}}, }, nil +{{ else }} + return &types.{{$func.MakeGoReturn}}Value{ + V: {{$func.ConvertStart}}{{ if not (eq $func.GolangPackage.Alias "") }}{{$func.GolangPackage.Alias}}{{else}}{{$func.GolangPackage.Name}}{{end}}.{{$func.GolangFunc}}({{$func.MakeGolangArgs}}{{$func.ConvertStop}}), + }, nil +{{ end -}} } {{ end -}} diff --git a/lang/funcs/funcgen/templates/generated_funcs_test.go.tpl b/lang/funcs/funcgen/templates/generated_funcs_test.go.tpl deleted file mode 100644 index 34fc2916..00000000 --- a/lang/funcs/funcgen/templates/generated_funcs_test.go.tpl +++ /dev/null @@ -1,41 +0,0 @@ -// Mgmt -// Copyright (C) 2013-2019+ James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -package core - -import ( - "testing" - - "github.com/purpleidea/mgmt/lang/types" -) -{{ range $i, $func := .Functions }} -func test{{$func.GoFunc}}(t *testing.T, {{$func.MakeTestSign}}) { - value, err := {{$func.GoFunc}}({{$func.TestInput}}) - if err != nil { - t.Error(err) - return - } - if value.{{$func.MakeGoReturn}}() != expected { - t.Errorf("invalid output, expected %s, got %s", expected, value.{{$func.MakeGoReturn}}()) - } -} -{{ range $index, $test := $func.Tests }} -func Test{{$func.GoFunc}}{{$index}}(t *testing.T) { - test{{$func.GoFunc}}(t, {{.MakeTestArgs}}) -} -{{ end -}} -{{ end -}} diff --git a/lang/interpret_test/TestAstFunc2/deploy-readfile0/main.mcl b/lang/interpret_test/TestAstFunc2/deploy-readfile0/main.mcl index fa341cf3..cbdb4e84 100644 --- a/lang/interpret_test/TestAstFunc2/deploy-readfile0/main.mcl +++ b/lang/interpret_test/TestAstFunc2/deploy-readfile0/main.mcl @@ -1,4 +1,4 @@ -import "strings" +import "golang/strings" import "deploy" import "second.mcl" import "mod1/" diff --git a/lang/interpret_test/TestAstFunc2/deploy-readfile1/main.mcl b/lang/interpret_test/TestAstFunc2/deploy-readfile1/main.mcl index fa341cf3..cbdb4e84 100644 --- a/lang/interpret_test/TestAstFunc2/deploy-readfile1/main.mcl +++ b/lang/interpret_test/TestAstFunc2/deploy-readfile1/main.mcl @@ -1,4 +1,4 @@ -import "strings" +import "golang/strings" import "deploy" import "second.mcl" import "mod1/"