lang: funcs: Add deploy package with readfile related functions

This adds a readfile function to actually access files from our deploy.
A fun side effect is that we can even access our own code! In general,
it's a good reminder that you should only run trusted code on your own
infrastructure. This also includes a fancy new test case.

lang/funcs/core/core.go

@@ -20,6 +20,7 @@ package core
 import (
 	// import so the funcs register
 	_ "github.com/purpleidea/mgmt/lang/funcs/core/datetime"
+	_ "github.com/purpleidea/mgmt/lang/funcs/core/deploy"
 	_ "github.com/purpleidea/mgmt/lang/funcs/core/example"
 	_ "github.com/purpleidea/mgmt/lang/funcs/core/example/nested"
 	_ "github.com/purpleidea/mgmt/lang/funcs/core/fmt"

lang/funcs/core/deploy/abspath_func.go

@@ -0,0 +1,156 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package coredeploy
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/purpleidea/mgmt/lang/funcs"
+	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/lang/types"
+)
+
+func init() {
+	funcs.ModuleRegister(moduleName, "abspath", func() interfaces.Func { return &AbsPathFunc{} }) // must register the func and name
+}
+
+const (
+	pathArg = "path"
+)
+
+// AbsPathFunc is a function that returns the absolute, full path in the deploy
+// from an input path that is relative to the calling file. If you pass it an
+// empty string, you'll just get the absolute deploy directory path that you're
+// in.
+type AbsPathFunc struct {
+	init *interfaces.Init
+	data *interfaces.FuncData
+	last types.Value // last value received to use for diff
+
+	path   string // the active path
+	result string // last calculated output
+
+	closeChan chan struct{}
+}
+
+// SetData is used by the language to pass our function some code-level context.
+func (obj *AbsPathFunc) SetData(data *interfaces.FuncData) {
+	obj.data = data
+}
+
+// ArgGen returns the Nth arg name for this function.
+func (obj *AbsPathFunc) ArgGen(index int) (string, error) {
+	seq := []string{pathArg}
+	if l := len(seq); index >= l {
+		return "", fmt.Errorf("index %d exceeds arg length of %d", index, l)
+	}
+	return seq[index], nil
+}
+
+// Validate makes sure we've built our struct properly. It is usually unused for
+// normal functions that users can use directly.
+func (obj *AbsPathFunc) Validate() error {
+	return nil
+}
+
+// Info returns some static info about itself.
+func (obj *AbsPathFunc) Info() *interfaces.Info {
+	return &interfaces.Info{
+		Pure: false, // maybe false because the file contents can change
+		Memo: false,
+		Sig:  types.NewType(fmt.Sprintf("func(%s str) str", pathArg)),
+	}
+}
+
+// Init runs some startup code for this function.
+func (obj *AbsPathFunc) Init(init *interfaces.Init) error {
+	obj.init = init
+	obj.closeChan = make(chan struct{})
+	if obj.data == nil {
+		// programming error
+		return fmt.Errorf("missing function data")
+	}
+	return nil
+}
+
+// Stream returns the changing values that this func has over time.
+func (obj *AbsPathFunc) Stream() error {
+	defer close(obj.init.Output) // the sender closes
+	for {
+		select {
+		case input, ok := <-obj.init.Input:
+			if !ok {
+				obj.init.Input = nil // don't infinite loop back
+				continue             // no more inputs, but don't return!
+			}
+			//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
+			//	return errwrap.Wrapf(err, "wrong function input")
+			//}
+
+			if obj.last != nil && input.Cmp(obj.last) == nil {
+				continue // value didn't change, skip it
+			}
+			obj.last = input // store for next
+
+			path := input.Struct()[pathArg].Str()
+			// TODO: add validation for absolute path?
+			if path == obj.path {
+				continue // nothing changed
+			}
+			obj.path = path
+
+			p := strings.TrimSuffix(obj.data.Base, "/")
+			if p == obj.data.Base { // didn't trim, so we fail
+				// programming error
+				return fmt.Errorf("no trailing slash on Base, got: `%s`", p)
+			}
+			result := p
+
+			if obj.path == "" {
+				result += "/" // add the above trailing slash back
+			} else if !strings.HasPrefix(obj.path, "/") {
+				return fmt.Errorf("path was not absolute, got: `%s`", obj.path)
+				//result += "/" // be forgiving ?
+			}
+			result += obj.path
+
+			if obj.result == result {
+				continue // result didn't change
+			}
+			obj.result = result // store new result
+
+		case <-obj.closeChan:
+			return nil
+		}
+
+		select {
+		case obj.init.Output <- &types.StrValue{
+			V: obj.result,
+		}:
+		case <-obj.closeChan:
+			return nil
+		}
+	}
+}
+
+// Close runs some shutdown code for this function and turns off the stream.
+func (obj *AbsPathFunc) Close() error {
+	close(obj.closeChan)
+	return nil
+}

lang/funcs/core/deploy/deploy.go

@@ -0,0 +1,23 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package coredeploy
+
+const (
+	// moduleName is the prefix given to all the functions in this module.
+	moduleName = "deploy"
+)

lang/funcs/core/deploy/readfile_func.go

@@ -0,0 +1,165 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package coredeploy
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/purpleidea/mgmt/lang/funcs"
+	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/lang/types"
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+func init() {
+	funcs.ModuleRegister(moduleName, "readfile", func() interfaces.Func { return &ReadFileFunc{} }) // must register the func and name
+}
+
+// ReadFileFunc is a function that reads the full contents from a file in our
+// deploy. The file contents can only change with a new deploy, so this is
+// static. Please note that this is different from the readfile function in the
+// os package.
+type ReadFileFunc struct {
+	init *interfaces.Init
+	data *interfaces.FuncData
+	last types.Value // last value received to use for diff
+
+	filename string // the active filename
+	result   string // last calculated output
+
+	closeChan chan struct{}
+}
+
+// SetData is used by the language to pass our function some code-level context.
+func (obj *ReadFileFunc) SetData(data *interfaces.FuncData) {
+	obj.data = data
+}
+
+// ArgGen returns the Nth arg name for this function.
+func (obj *ReadFileFunc) ArgGen(index int) (string, error) {
+	seq := []string{"filename"}
+	if l := len(seq); index >= l {
+		return "", fmt.Errorf("index %d exceeds arg length of %d", index, l)
+	}
+	return seq[index], nil
+}
+
+// Validate makes sure we've built our struct properly. It is usually unused for
+// normal functions that users can use directly.
+func (obj *ReadFileFunc) Validate() error {
+	return nil
+}
+
+// Info returns some static info about itself.
+func (obj *ReadFileFunc) Info() *interfaces.Info {
+	return &interfaces.Info{
+		Pure: false, // maybe false because the file contents can change
+		Memo: false,
+		Sig:  types.NewType("func(filename str) str"),
+	}
+}
+
+// Init runs some startup code for this function.
+func (obj *ReadFileFunc) Init(init *interfaces.Init) error {
+	obj.init = init
+	obj.closeChan = make(chan struct{})
+	if obj.data == nil {
+		// programming error
+		return fmt.Errorf("missing function data")
+	}
+	return nil
+}
+
+// Stream returns the changing values that this func has over time.
+func (obj *ReadFileFunc) Stream() error {
+	defer close(obj.init.Output) // the sender closes
+	for {
+		select {
+		case input, ok := <-obj.init.Input:
+			if !ok {
+				obj.init.Input = nil // don't infinite loop back
+				continue             // no more inputs, but don't return!
+			}
+			//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
+			//	return errwrap.Wrapf(err, "wrong function input")
+			//}
+
+			if obj.last != nil && input.Cmp(obj.last) == nil {
+				continue // value didn't change, skip it
+			}
+			obj.last = input // store for next
+
+			filename := input.Struct()["filename"].Str()
+			// TODO: add validation for absolute path?
+			if filename == obj.filename {
+				continue // nothing changed
+			}
+			obj.filename = filename
+
+			p := strings.TrimSuffix(obj.data.Base, "/")
+			if p == obj.data.Base { // didn't trim, so we fail
+				// programming error
+				return fmt.Errorf("no trailing slash on Base, got: `%s`", p)
+			}
+			path := p
+
+			if !strings.HasPrefix(obj.filename, "/") {
+				return fmt.Errorf("filename was not absolute, got: `%s`", obj.filename)
+				//path += "/" // be forgiving ?
+			}
+			path += obj.filename
+
+			fs, err := obj.init.World.Fs(obj.data.FsURI) // open the remote file system
+			if err != nil {
+				return errwrap.Wrapf(err, "can't load code from file system `%s`", obj.data.FsURI)
+			}
+			// this is relative to the module dir the func is in!
+			content, err := fs.ReadFile(path) // open the remote file system
+			// We could use it directly, but it feels like less correct.
+			//content, err := obj.data.Fs.ReadFile(path) // open the remote file system
+			if err != nil {
+				return errwrap.Wrapf(err, "can't read file `%s` (%s)", obj.filename, path)
+			}
+
+			result := string(content) // convert to string
+
+			if obj.result == result {
+				continue // result didn't change
+			}
+			obj.result = result // store new result
+
+		case <-obj.closeChan:
+			return nil
+		}
+
+		select {
+		case obj.init.Output <- &types.StrValue{
+			V: obj.result,
+		}:
+		case <-obj.closeChan:
+			return nil
+		}
+	}
+}
+
+// Close runs some shutdown code for this function and turns off the stream.
+func (obj *ReadFileFunc) Close() error {
+	close(obj.closeChan)
+	return nil
+}

lang/funcs/core/deploy/readfileabs_func.go

@@ -0,0 +1,151 @@
+// Mgmt
+// Copyright (C) 2013-2019+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package coredeploy
+
+import (
+	"fmt"
+
+	"github.com/purpleidea/mgmt/lang/funcs"
+	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/lang/types"
+	"github.com/purpleidea/mgmt/util/errwrap"
+)
+
+func init() {
+	funcs.ModuleRegister(moduleName, "readfileabs", func() interfaces.Func { return &ReadFileAbsFunc{} }) // must register the func and name
+}
+
+// ReadFileAbsFunc is a function that reads the full contents from a file in our
+// deploy. The file contents can only change with a new deploy, so this is
+// static. In particular, this takes an absolute path relative to the root
+// deploy. In general, you should use `deploy.readfile` instead. Please note
+// that this is different from the readfile function in the os package.
+type ReadFileAbsFunc struct {
+	init *interfaces.Init
+	data *interfaces.FuncData
+	last types.Value // last value received to use for diff
+
+	filename string // the active filename
+	result   string // last calculated output
+
+	closeChan chan struct{}
+}
+
+// SetData is used by the language to pass our function some code-level context.
+func (obj *ReadFileAbsFunc) SetData(data *interfaces.FuncData) {
+	obj.data = data
+}
+
+// ArgGen returns the Nth arg name for this function.
+func (obj *ReadFileAbsFunc) ArgGen(index int) (string, error) {
+	seq := []string{"filename"}
+	if l := len(seq); index >= l {
+		return "", fmt.Errorf("index %d exceeds arg length of %d", index, l)
+	}
+	return seq[index], nil
+}
+
+// Validate makes sure we've built our struct properly. It is usually unused for
+// normal functions that users can use directly.
+func (obj *ReadFileAbsFunc) Validate() error {
+	return nil
+}
+
+// Info returns some static info about itself.
+func (obj *ReadFileAbsFunc) Info() *interfaces.Info {
+	return &interfaces.Info{
+		Pure: false, // maybe false because the file contents can change
+		Memo: false,
+		Sig:  types.NewType("func(filename str) str"),
+	}
+}
+
+// Init runs some startup code for this function.
+func (obj *ReadFileAbsFunc) Init(init *interfaces.Init) error {
+	obj.init = init
+	obj.closeChan = make(chan struct{})
+	if obj.data == nil {
+		// programming error
+		return fmt.Errorf("missing function data")
+	}
+	return nil
+}
+
+// Stream returns the changing values that this func has over time.
+func (obj *ReadFileAbsFunc) Stream() error {
+	defer close(obj.init.Output) // the sender closes
+	for {
+		select {
+		case input, ok := <-obj.init.Input:
+			if !ok {
+				obj.init.Input = nil // don't infinite loop back
+				continue             // no more inputs, but don't return!
+			}
+			//if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil {
+			//	return errwrap.Wrapf(err, "wrong function input")
+			//}
+
+			if obj.last != nil && input.Cmp(obj.last) == nil {
+				continue // value didn't change, skip it
+			}
+			obj.last = input // store for next
+
+			filename := input.Struct()["filename"].Str()
+			// TODO: add validation for absolute path?
+			if filename == obj.filename {
+				continue // nothing changed
+			}
+			obj.filename = filename
+
+			fs, err := obj.init.World.Fs(obj.data.FsURI) // open the remote file system
+			if err != nil {
+				return errwrap.Wrapf(err, "can't load code from file system `%s`", obj.data.FsURI)
+			}
+			content, err := fs.ReadFile(obj.filename) // open the remote file system
+			// We could use it directly, but it feels like less correct.
+			//content, err := obj.data.Fs.ReadFile(obj.filename) // open the remote file system
+			if err != nil {
+				return errwrap.Wrapf(err, "can't read file `%s`", obj.filename)
+			}
+
+			result := string(content) // convert to string
+
+			if obj.result == result {
+				continue // result didn't change
+			}
+			obj.result = result // store new result
+
+		case <-obj.closeChan:
+			return nil
+		}
+
+		select {
+		case obj.init.Output <- &types.StrValue{
+			V: obj.result,
+		}:
+		case <-obj.closeChan:
+			return nil
+		}
+	}
+}
+
+// Close runs some shutdown code for this function and turns off the stream.
+func (obj *ReadFileAbsFunc) Close() error {
+	close(obj.closeChan)
+	return nil
+}

lang/funcs/core/os/readfile_func.go

@@ -35,6 +35,8 @@ func init() {
 
 // ReadFileFunc is a function that reads the full contents from a local file. If
 // the file contents change or the file path changes, a new string will be sent.
+// Please note that this is different from the readfile function in the deploy
+// package.
 type ReadFileFunc struct {
 	init *interfaces.Init
 	last types.Value // last value received to use for diff

lang/interpret_test/TestAstFunc2/deploy-readfile0.output

@@ -0,0 +1,3 @@
+Vertex: test[This is file1 in the files/ folder.]
+Vertex: test[This is file2 in the files/ folder.]
+Vertex: test[This is file3 in the files/ folder inside of the mod1/ module.]

lang/interpret_test/TestAstFunc2/deploy-readfile0/files/file1

@@ -0,0 +1 @@
+This is file1 in the files/ folder.

lang/interpret_test/TestAstFunc2/deploy-readfile0/files/file2

@@ -0,0 +1 @@
+This is file2 in the files/ folder.

lang/interpret_test/TestAstFunc2/deploy-readfile0/main.mcl

@@ -0,0 +1,53 @@
+import "strings"
+import "deploy"
+import "second.mcl"
+import "mod1/"
+
+#$f1 = "/metadata.yaml" # works
+#$f1 = "/main.mcl" # works
+$f1 = "/files/file1"
+
+$f2 = "/files/file2"
+
+$f3 = "/mod1/files/file3"
+
+# the abspath method shouldn't be used often, it's here for testing...
+if $f1 != deploy.abspath($f1) { # should be the same, since we're in the same dir
+	test "f1 error" {}
+}
+if $f2 != $second.f2 {
+	test "f2 error" {}
+}
+if $f3 != $mod1.f3 {
+	test "f3 error" {}
+}
+
+# the readfileabs method shouldn't be used often, it's here for testing...
+$x1 = deploy.readfileabs($f1)
+$x2 = deploy.readfileabs($f2)
+$x3 = deploy.readfileabs($f3)
+
+if $x1 != deploy.readfile($f1) {
+	test "x1 error" {}
+}
+if $x2 != $second.x2 {
+	test "x2 error" {}
+}
+if $x3 != $mod1.x3 {
+	test "x3 error" {}
+}
+
+# hide the newlines from our output
+test strings.trim_space($x1) {}
+test strings.trim_space($x2) {}
+test strings.trim_space($x3) {}
+# debugging:
+#test "f1" {
+#	anotherstr => $x1,
+#}
+#test "f2" {
+#	anotherstr => $x2,
+#}
+#test "f3" {
+#	anotherstr => $x3,
+#}

lang/interpret_test/TestAstFunc2/deploy-readfile0/metadata.yaml

@@ -0,0 +1 @@
+#files: "files/"	# these are some extra files we can use (is the default)

lang/interpret_test/TestAstFunc2/deploy-readfile0/mod1/files/file3

@@ -0,0 +1 @@
+This is file3 in the files/ folder inside of the mod1/ module.

lang/interpret_test/TestAstFunc2/deploy-readfile0/mod1/main.mcl

@@ -0,0 +1,6 @@
+import "deploy"
+
+# relative paths for us
+$f = "/files/file3" # real file is in: /mod1/files/file3
+$f3 = deploy.abspath($f)
+$x3 = deploy.readfile($f)

lang/interpret_test/TestAstFunc2/deploy-readfile0/mod1/metadata.yaml

@@ -0,0 +1 @@
+#files: "files/"	# these are some extra files we can use (is the default)

lang/interpret_test/TestAstFunc2/deploy-readfile0/second.mcl

@@ -0,0 +1,6 @@
+import "deploy"
+
+# relative paths for us
+$f = "/files/file2" # real file is here as well
+$f2 = deploy.abspath($f)
+$x2 = deploy.readfile($f)

lang/interpret_test/TestAstFunc2/deploy-readfile1.output

@@ -0,0 +1,3 @@
+Vertex: test[This is a different file1 in the files/ folder.]
+Vertex: test[This is a different file2 in the files/ folder.]
+Vertex: test[This is a different file3 in the files/ folder inside of the mod1/ module.]

lang/interpret_test/TestAstFunc2/deploy-readfile1/files/file1

@@ -0,0 +1 @@
+This is a different file1 in the files/ folder.

lang/interpret_test/TestAstFunc2/deploy-readfile1/files/file2

@@ -0,0 +1 @@
+This is a different file2 in the files/ folder.

lang/interpret_test/TestAstFunc2/deploy-readfile1/main.mcl

@@ -0,0 +1,53 @@
+import "strings"
+import "deploy"
+import "second.mcl"
+import "mod1/"
+
+#$f1 = "/metadata.yaml" # works
+#$f1 = "/main.mcl" # works
+$f1 = "/files/file1"
+
+$f2 = "/files/file2"
+
+$f3 = "/mod1/files/file3"
+
+# the abspath method shouldn't be used often, it's here for testing...
+if $f1 != deploy.abspath($f1) { # should be the same, since we're in the same dir
+	test "f1 error" {}
+}
+if $f2 != $second.f2 {
+	test "f2 error" {}
+}
+if $f3 != $mod1.f3 {
+	test "f3 error" {}
+}
+
+# the readfileabs method shouldn't be used often, it's here for testing...
+$x1 = deploy.readfileabs($f1)
+$x2 = deploy.readfileabs($f2)
+$x3 = deploy.readfileabs($f3)
+
+if $x1 != deploy.readfile($f1) {
+	test "x1 error" {}
+}
+if $x2 != $second.x2 {
+	test "x2 error" {}
+}
+if $x3 != $mod1.x3 {
+	test "x3 error" {}
+}
+
+# hide the newlines from our output
+test strings.trim_space($x1) {}
+test strings.trim_space($x2) {}
+test strings.trim_space($x3) {}
+# debugging:
+#test "f1" {
+#	anotherstr => $x1,
+#}
+#test "f2" {
+#	anotherstr => $x2,
+#}
+#test "f3" {
+#	anotherstr => $x3,
+#}

lang/interpret_test/TestAstFunc2/deploy-readfile1/metadata.yaml

@@ -0,0 +1 @@
+#files: "files/"	# these are some extra files we can use (is the default)

lang/interpret_test/TestAstFunc2/deploy-readfile1/mod1/files/file3

@@ -0,0 +1 @@
+This is a different file3 in the files/ folder inside of the mod1/ module.

lang/interpret_test/TestAstFunc2/deploy-readfile1/mod1/main.mcl

@@ -0,0 +1,6 @@
+import "deploy"
+
+# relative paths for us
+$f = "/files/file3" # real file is in: /mod1/files/file3
+$f3 = deploy.abspath($f)
+$x3 = deploy.readfile($f)

lang/interpret_test/TestAstFunc2/deploy-readfile1/mod1/metadata.yaml

@@ -0,0 +1 @@
+#files: "files/"	# these are some extra files we can use (is the default)

lang/interpret_test/TestAstFunc2/deploy-readfile1/second.mcl

@@ -0,0 +1,6 @@
+import "deploy"
+
+# relative paths for us
+$f = "/files/file2" # real file is here as well
+$f2 = deploy.abspath($f)
+$x2 = deploy.readfile($f)