lang: Replace the go-bindata usage with embed

This doesn't let us have nested mcl at the moment, but we could improve
on this with an embed API for each package. For now this makes building
the project easier.

.github/workflows/test.yaml

@@ -37,9 +37,8 @@ jobs:
       #fail-fast: false
 
     steps:
-      # Do not shallow fetch, will fail when building bindata/
+      # Do not shallow fetch. The path can't be absolute, so we need to move it
-      # The path can't be absolute, so we need to move it to the
+      # to the expected location later.
-      # expected location later.
       - name: Clone mgmt
         uses: actions/checkout@v2
         with:

Makefile

@@ -16,12 +16,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 SHELL = /usr/bin/env bash
-.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs
+.PHONY: all art cleanart version program lang path deps run race generate build build-debug crossbuild clean test gofmt yamlfmt format docs
 .PHONY: rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag
 .PHONY: mkosi mkosi_fedora-30 mkosi_fedora-29 mkosi_centos-7 mkosi_debian-10 mkosi_ubuntu-bionic mkosi_archlinux
 .PHONY: release releases_path release_fedora-30 release_fedora-29 release_centos-7 release_debian-10 release_ubuntu-bionic release_archlinux
 .PHONY: funcgen
-.SILENT: clean bindata
+.SILENT: clean
 
 # a large amount of output from this `find`, can cause `make` to be much slower!
 GO_FILES := $(shell find * -name '*.go' -not -path 'old/*' -not -path 'tmp/*')
@@ -137,10 +137,6 @@ run: ## run mgmt
 race:
 	find . -maxdepth 1 -type f -name '*.go' -not -name '*_test.go' | xargs go run -race -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)"
 
-# generate go files from non-go source
-bindata: ## generate go files from non-go sources
-	$(MAKE) --quiet -C lang/funcs
-
 generate:
 	go generate
 
@@ -167,7 +163,7 @@ build-debug: $(PROGRAM)
 # extract os and arch from target pattern
 GOOS=$(firstword $(subst -, ,$*))
 GOARCH=$(lastword $(subst -, ,$*))
-build/mgmt-%: $(GO_FILES) $(MCL_FILES) | bindata lang funcgen
+build/mgmt-%: $(GO_FILES) $(MCL_FILES) | lang funcgen
 	@echo "Building: $(PROGRAM), os/arch: $*, version: $(SVERSION)..."
 	@time env GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags=$(PKGNAME)="-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS)
 

lang/ast/structs.go

@@ -30,7 +30,7 @@ import (
 	"github.com/purpleidea/mgmt/engine"
 	engineUtil "github.com/purpleidea/mgmt/engine/util"
 	"github.com/purpleidea/mgmt/lang/funcs"
-	"github.com/purpleidea/mgmt/lang/funcs/bindata"
+	"github.com/purpleidea/mgmt/lang/funcs/core"
 	"github.com/purpleidea/mgmt/lang/funcs/structs"
 	"github.com/purpleidea/mgmt/lang/inputs"
 	"github.com/purpleidea/mgmt/lang/interfaces"
@@ -3057,7 +3057,7 @@ func (obj *StmtProg) importScope(info *interfaces.ImportData, scope *interfaces.
 // importSystemScope takes the name of a built-in system scope (eg: "fmt") and
 // returns the scope struct for that built-in. This function is slightly less
 // trivial than expected, because the scope is built from both native mcl code
-// and golang code as well. The native mcl code is compiled in as bindata.
+// and golang code as well. The native mcl code is compiled in with "embed".
 // TODO: can we memoize?
 func (obj *StmtProg) importSystemScope(name string) (*interfaces.Scope, error) {
 	// this basically loop through the registeredFuncs and includes
@@ -3096,7 +3096,10 @@ func (obj *StmtProg) importSystemScope(name string) (*interfaces.Scope, error) {
 	// to the remote machines as well, and we want to load off of it...
 
 	// now add any compiled-in mcl code
-	paths := bindata.AssetNames()
+	paths, err := core.AssetNames()
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "can't read asset names")
+	}
 	// results are not sorted by default (ascertained by reading the code!)
 	sort.Strings(paths)
 	newScope := interfaces.EmptyScope()
@@ -3113,7 +3116,7 @@ func (obj *StmtProg) importSystemScope(name string) (*interfaces.Scope, error) {
 			continue
 		}
 
-		b, err := bindata.Asset(p)
+		b, err := core.Asset(p)
 		if err != nil {
 			return nil, errwrap.Wrapf(err, "can't read asset: `%s`", p)
 		}

lang/funcs/Makefile

@@ -1,45 +0,0 @@
-# Mgmt
-# Copyright (C) 2013-2022+ James Shubin and the project contributors
-# Written by James Shubin <james@shubin.ca> and the project contributors
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# The bindata target generates go files from any source defined below. To use
-# the files, import the generated "bindata" package and use:
-# `bytes, err := bindata.Asset("FILEPATH")`
-# where FILEPATH is the path of the original input file relative to `bindata/`.
-# To get a list of files stored in this "bindata" package, you can use:
-# `paths := bindata.AssetNames()` and `paths, err := bindata.AssetDir(name)`
-# to get a list of files with a directory prefix.
-
-.PHONY: build clean
-default: build
-
-MCL_FILES := $(shell find * -name '*.mcl' -not -path 'old/*' -not -path 'tmp/*')
-GENERATED = bindata/bindata.go
-
-build: $(GENERATED)
-
-# add more input files as dependencies at the end here...
-$(GENERATED): $(MCL_FILES)
-	@echo "Generating: native mcl..."
-	@# go-bindata --pkg bindata -o <OUTPUT> <INPUT>
-	go-bindata --pkg bindata -o ./$@ $^
-	@# gofmt the output file
-	gofmt -s -w $@
-	@ROOT=$$(dirname "$${BASH_SOURCE}")/../.. && $$ROOT/misc/header.sh '$@'
-
-clean:
-	@# remove generated bindata/bindata.go
-	@ROOT=$$(dirname "$${BASH_SOURCE}")/../.. && rm -f $(GENERATED)

lang/funcs/bindata.mcl

@@ -1,2 +0,0 @@
-# You can add *.mcl files alongside the *.go files into the core/ directory.
-# They will get compiled into the binary when it is built.

lang/funcs/bindata/.gitignore

@@ -1,2 +0,0 @@
-# this file gets generated here
-bindata.go

lang/funcs/bindata/doc.go

@@ -1,19 +0,0 @@
-// Mgmt
-// Copyright (C) 2013-2022+ James Shubin and the project contributors
-// Written by James Shubin <james@shubin.ca> and the project contributors
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-// Package bindata stores core mcl code that is built-in at compile time.
-package bindata

lang/funcs/core/core.go

@@ -18,6 +18,9 @@
 package core
 
 import (
+	"embed"
+	"io/fs"
+
 	// import so the funcs register
 	_ "github.com/purpleidea/mgmt/lang/funcs/core/convert"
 	_ "github.com/purpleidea/mgmt/lang/funcs/core/datetime"
@@ -34,3 +37,34 @@ import (
 	_ "github.com/purpleidea/mgmt/lang/funcs/core/sys"
 	_ "github.com/purpleidea/mgmt/lang/funcs/core/world"
 )
+
+// TODO: Instead of doing this one-level embed, we could give each package an
+// API that it calls to "register" the private embed.FS that it wants to share.
+
+//go:embed */*.mcl
+var mcl embed.FS
+
+// AssetNames returns a flattened list of embedded .mcl file paths.
+func AssetNames() ([]string, error) {
+	fileSystem := mcl
+	paths := []string{}
+	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() { // skip the dirs
+			return nil
+		}
+		paths = append(paths, path)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return paths, nil
+}
+
+// Asset returns the contents of an embedded .mcl file.
+func Asset(name string) ([]byte, error) {
+	return mcl.ReadFile(name)
+}

lang/funcs/funcs.go

@@ -41,7 +41,7 @@ const (
 	// NOTE: the template library will panic if it is one of: .-#
 	ReplaceChar = "_"
 
-	// CoreDir is the directory prefix where core bindata mcl code is added.
+	// CoreDir is the directory prefix where core mcl code is embedded.
 	CoreDir = "core/"
 )
 

misc/make-deps.sh

@@ -150,7 +150,6 @@ go get golang.org/x/tools/cmd/goyacc			# formerly `go tool yacc`
 go get golang.org/x/tools/cmd/stringer			# for automatic stringer-ing
 go get golang.org/x/lint/golint				# for `golint`-ing
 go get golang.org/x/tools/cmd/goimports		# for fmt
-go get github.com/kevinburke/go-bindata/go-bindata	# for compiling in non golang files
 go get github.com/dvyukov/go-fuzz/go-fuzz		# for fuzzing the mcl lang bits
 if in_ci; then
 	go get -u gopkg.in/alecthomas/gometalinter.v1 && \

test/test-golint.sh

@@ -27,7 +27,7 @@ if [ "$COMMITS" != "" ] && [ "$COMMITS" -gt "1" ]; then
 fi
 
 # find all go files, exluding temporary directories and generated files
-LINT=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'bindata/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'lang/interpolate/parse.generated.go' -not -path 'lang/types/kind_stringer.go' -not -path 'vendor/*' -exec golint {} \;)	# current golint output
+LINT=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'lang/interpolate/parse.generated.go' -not -path 'lang/types/kind_stringer.go' -not -path 'vendor/*' -exec golint {} \;)	# current golint output
 
 COUNT=`echo -e "$LINT" | wc -l`	# number of golint problems in current branch
 [ "$LINT" = "" ] && echo PASS && exit	# everything is "perfect"
@@ -55,7 +55,7 @@ while read -r line; do
 done <<< "$NUMSTAT1"	# three < is the secret to putting a variable into read
 
 git checkout "$PREVIOUS" &>/dev/null	# previous commit
-LINT1=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'bindata/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'vendor/*' -exec golint {} \;)
+LINT1=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'vendor/*' -exec golint {} \;)
 COUNT1=`echo -e "$LINT1" | wc -l`	# number of golint problems in older branch
 
 # clean up

test/test-gometalinter.sh

@@ -47,7 +47,6 @@ gml="$gml --enable=misspell"
 # TODO: at least until https://github.com/alecthomas/gometalinter/issues/270
 gml="$gml --exclude=lang/parser/lexer.nn.go"
 gml="$gml --exclude=lang/parser/y.go"
-gml="$gml --exclude=bindata/bindata.go"
 gml="$gml --exclude=lang/types/kind_stringer.go"
 gml="$gml --exclude=lang/interpolate/parse.generated.go"
 

test/test-govet.sh

@@ -60,10 +60,6 @@ function naked-error() {
 
 # catch errors that start with a capital
 function lowercase-errors() {
-	if [[ $1 == *"/bindata.go" ]]; then	# ends with bindata.go ?
-		return 0	# skip those generated files
-	fi
-
 	if grep -E 'errors\.New\(\"[A-Z]' "$1"; then
 		return 1
 	fi
@@ -101,10 +97,6 @@ function consistent-imports() {
 }
 
 function reflowed-comments() {
-	if [[ $1 == *"/bindata.go" ]]; then	# ends with bindata.go ?
-		return 0	# skip those generated files
-	fi
-
 	if [ "$1" = './lang/funcs/core/generated_funcs.go' ]; then
 		return 0
 	fi